test: regression test all files (#325)

* test: regression test for all regression files

* refactor: extract assert committer flow function

.github/workflows/ci.yml

@@ -110,7 +110,7 @@ jobs:
           credentials_json: ${{ secrets.COMMITER_PRODUCTS_EXT_WRITER_JSON }}
       - uses: 'google-github-actions/setup-gcloud@v2'
       - run: echo "BENCH_INPUT_FILES_PREFIX=$(cat ./crates/committer_cli/src/tests/flow_test_files_prefix)" >> $GITHUB_ENV
-      - run: gcloud storage cp -r gs://committer-testing-artifacts/$BENCH_INPUT_FILES_PREFIX/* ./crates/committer_cli/benches
+      - run: gcloud storage cp -r gs://committer-testing-artifacts/$BENCH_INPUT_FILES_PREFIX/* ./crates/committer_cli/test_inputs
       - run: cargo test --release -- --include-ignored test_regression
 
   benchmarking:
@@ -139,8 +139,8 @@ jobs:
       - run: cargo bench
 
       # Backup the downloaded files to avoid re-downloading them if they didn't change (overwritten by checkout).
-      - run: mv ./crates/committer_cli/benches/tree_flow_inputs.json ./crates/committer_cli/benches/tree_flow_inputs.json_bu
-      - run: mv ./crates/committer_cli/benches/committer_flow_inputs.json ./crates/committer_cli/benches/committer_flow_inputs.json_bu
+      #- run: mv ./crates/committer_cli/test_inputs/tree_flow_inputs.json ./crates/committer_cli/test_inputs/tree_flow_inputs.json_bu
+      #- run: mv ./crates/committer_cli/test_inputs/committer_flow_inputs.json ./crates/committer_cli/test_inputs/committer_flow_inputs.json_bu
 
       # Checkout the new code.
       - uses: actions/checkout@v4
@@ -149,15 +149,15 @@ jobs:
       - run: echo "NEW_BENCH_INPUT_FILES_PREFIX=$(cat ./crates/committer_cli/src/tests/flow_test_files_prefix)" >> $GITHUB_ENV
 
       # Input files didn't change.
-      - if: env.OLD_BENCH_INPUT_FILES_PREFIX == env.NEW_BENCH_INPUT_FILES_PREFIX
-        run: |
-          mv ./crates/committer_cli/benches/tree_flow_inputs.json_bu ./crates/committer_cli/benches/tree_flow_inputs.json
-          mv ./crates/committer_cli/benches/committer_flow_inputs.json_bu ./crates/committer_cli/benches/committer_flow_inputs.json
+      #- if: env.OLD_BENCH_INPUT_FILES_PREFIX == env.NEW_BENCH_INPUT_FILES_PREFIX
+      #  run: |
+      #    mv ./crates/committer_cli/test_inputs/tree_flow_inputs.json_bu ./crates/committer_cli/test_inputs/tree_flow_inputs.json
+      #    mv ./crates/committer_cli/test_inputs/committer_flow_inputs.json_bu ./crates/committer_cli/test_inputs/committer_flow_inputs.json
 
       # Input files did change, download new inputs.
-      - if: env.OLD_BENCH_INPUT_FILES_PREFIX != env.NEW_BENCH_INPUT_FILES_PREFIX
-        run: |
-          gcloud storage cp -r gs://committer-testing-artifacts/$NEW_BENCH_INPUT_FILES_PREFIX/* ./crates/committer_cli/benches
+      #- if: env.OLD_BENCH_INPUT_FILES_PREFIX != env.NEW_BENCH_INPUT_FILES_PREFIX
+      - run: |
+          gcloud storage cp -r gs://committer-testing-artifacts/$NEW_BENCH_INPUT_FILES_PREFIX/* ./crates/committer_cli/test_inputs
 
       # Benchmark the new code, splitting the benchmarks, and prepare the results for posting a comment.
       - run: bash ./crates/committer_cli/benches/bench_split_and_prepare_post.sh benchmarks_list.txt bench_new.txt

crates/committer_cli/benches/committer_bench.rs

@@ -20,8 +20,8 @@ use committer_cli::{commands::parse_and_commit, tests::utils::parse_from_python:
 use criterion::{criterion_group, criterion_main, Criterion};
 
 const CONCURRENCY_MODE: bool = true;
-const SINGLE_TREE_FLOW_INPUT: &str = include_str!("tree_flow_inputs.json");
-const FLOW_TEST_INPUT: &str = include_str!("committer_flow_inputs.json");
+const SINGLE_TREE_FLOW_INPUT: &str = include_str!("../test_inputs/tree_flow_inputs.json");
+const FLOW_TEST_INPUT: &str = include_str!("../test_inputs/committer_flow_inputs.json");
 const OUTPUT_PATH: &str = "benchmark_output.txt";
 
 pub fn single_tree_flow_benchmark(criterion: &mut Criterion) {

crates/committer_cli/src/tests/regression_tests.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, fs};
 
 use committer::{
     block_committer::input::{ConfigImpl, Input},
@@ -19,9 +19,10 @@ use super::utils::parse_from_python::parse_input_single_storage_tree_flow_test;
 //2. Fix the max time threshold to be the expected time for the benchmark test.
 const MAX_TIME_FOR_SINGLE_TREE_BECHMARK_TEST: f64 = 5.0;
 const MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST: f64 = 5.0;
-const SINGLE_TREE_FLOW_INPUT: &str = include_str!("../../benches/tree_flow_inputs.json");
-const FLOW_TEST_INPUT: &str = include_str!("../../benches/committer_flow_inputs.json");
+const SINGLE_TREE_FLOW_INPUT: &str = include_str!("../../test_inputs/tree_flow_inputs.json");
+const FLOW_TEST_INPUT: &str = include_str!("../../test_inputs/committer_flow_inputs.json");
 const OUTPUT_PATH: &str = "benchmark_output.txt";
+const EXPECTED_NUMBER_OF_FILES: usize = 100;
 
 #[derive(derive_more::Deref)]
 struct FactMap(Map<String, Value>);
@@ -100,7 +101,7 @@ impl<'de> Deserialize<'de> for TreeRegressionInput {
     }
 }
 
-#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."]
+#[ignore = "To avoid running the regression test in Coverage or without the --release flag."]
 #[tokio::test(flavor = "multi_thread")]
 pub async fn test_regression_single_tree() {
     let TreeRegressionInput {
@@ -135,19 +136,17 @@ pub async fn test_regression_single_tree() {
     assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_SINGLE_TREE_BECHMARK_TEST);
 }
 
-#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."]
-#[tokio::test(flavor = "multi_thread")]
-pub async fn test_regression_committer_flow() {
+pub async fn test_single_committer_flow(input: &str, output_path: &str) {
     let CommitterRegressionInput {
         committer_input,
         contract_states_root: expected_contract_states_root,
         contract_classes_root: expected_contract_classes_root,
         expected_facts,
-    } = serde_json::from_str(FLOW_TEST_INPUT).unwrap();
+    } = serde_json::from_str(input).unwrap();
 
     let start = std::time::Instant::now();
     // Benchmark the committer flow test.
-    commit(committer_input.0, OUTPUT_PATH.to_owned()).await;
+    commit(committer_input.0, output_path.to_owned()).await;
     let execution_time = std::time::Instant::now() - start;
 
     // Assert correctness of the output of the committer flow test.
@@ -157,7 +156,7 @@ pub async fn test_regression_committer_flow() {
         storage: StorageObject {
             storage: Value::Object(storage_changes),
         },
-    } = serde_json::from_str(&std::fs::read_to_string(OUTPUT_PATH).unwrap()).unwrap()
+    } = serde_json::from_str(&std::fs::read_to_string(output_path).unwrap()).unwrap()
     else {
         panic!("Expected the storage to be an object.");
     };
@@ -169,3 +168,28 @@ pub async fn test_regression_committer_flow() {
     // Assert the execution time does not exceed the threshold.
     assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST);
 }
+#[ignore = "To avoid running the regression test in Coverage or without the --release flag."]
+#[tokio::test(flavor = "multi_thread")]
+pub async fn test_regression_committer_flow() {
+    test_single_committer_flow(FLOW_TEST_INPUT, OUTPUT_PATH).await;
+}
+
+#[ignore = "To avoid running the regression test in Coverage or without the --release flag."]
+#[tokio::test(flavor = "multi_thread")]
+pub async fn test_regression_committer_all_files() {
+    assert_eq!(
+        fs::read_dir("./test_inputs/regression_files")
+            .unwrap()
+            .count(),
+        EXPECTED_NUMBER_OF_FILES
+    );
+    let dir_path = fs::read_dir("./test_inputs/regression_files").unwrap();
+    for file_path in dir_path {
+        // TODO(Aner, 23/07/24): multi-thread the test.
+        test_single_committer_flow(
+            &fs::read_to_string(file_path.unwrap().path()).unwrap(),
+            OUTPUT_PATH,
+        )
+        .await;
+    }
+}