Benchmarks table

.gitignore

@@ -12,8 +12,7 @@ Cargo.lock
 **/proptest-regressions
 /output.txt
 flamegraph.svg
-**/*.trace
-**/*.memory
+dhat-heap.json
 
 **/.DS_Store
 **/*.swp

Cargo.toml

@@ -23,6 +23,7 @@ rayon = { version = "1.7.0", optional = true }
 giza-core = { git = "https://github.com/lambdaclass/giza", branch = "remove_prints", optional = true }
 giza-prover = { git = "https://github.com/lambdaclass/giza", branch = "remove_prints", optional = true }
 giza-runner = { git = "https://github.com/lambdaclass/giza", branch = "remove_prints", optional = true }
+dhat = { version = "0.3.2", optional = true }
 
 [dev-dependencies]
 proptest = "1.2.0"
@@ -39,21 +40,23 @@ instruments = []                   # This enables timing prints in prover and ve
 metal = ["lambdaworks-math/metal"]
 parallel = ["dep:rayon"]
 giza = ["dep:giza-core", "dep:giza-prover", "dep:giza-runner"]
+heap_profiling = ["dep:dhat"]
 
 [[bench]]
 name = "criterion_prover"
 harness = false
-metal = ["lambdaworks-math/metal"]
 
 [[bench]]
 name = "criterion_prover_70k"
 harness = false
-metal = ["lambdaworks-math/metal"]
+
+[[bench]]
+name = "criterion_prover_security"
+harness = false
 
 [[bench]]
 name = "criterion_verifier"
 harness = false
-metal = ["lambdaworks-math/metal"]
 
 [[bench]]
 name = "criterion_verifier_70k"
@@ -63,11 +66,24 @@ harness = false
 name = "criterion_giza"
 harness = false
 
+[[bench]]
+name = "criterion_verifier_security"
+harness = false
+
+[[bench]]
+name = "criterion_table"
+harness = false
+
+[[bench]]
+name = "dhat_prover"
+harness = false
+required-features = ["heap_profiling"]
 
 [profile.release]
 lto = true
 opt-level = 3
 codegen-units = 1
+debug = 1
 
 [profile.test]
 lto = "thin"

Makefile

@@ -19,10 +19,10 @@ build:
 	cargo build --release
 
 prove: build
-	cargo run --release prove $(PROGRAM_PATH) $(PROOF_PATH)
+	cargo run --release prove $(PROGRAM_PATH) $(PROOF_PATH) -s
 
 verify: build
-	cargo run --release verify $(PROOF_PATH)
+	cargo run --release verify $(PROOF_PATH) -s
 
 run_all: build
 	cargo run --release prove_and_verify $(PROGRAM_PATH)
@@ -43,12 +43,21 @@ clippy:
 	cargo clippy --workspace --all-targets -- -D warnings
 
 benchmarks_sequential: $(COMPILED_CAIRO0_PROGRAMS)
-	cargo bench
+	cargo bench --bench criterion_prover
+	cargo bench --bench criterion_verifier
 
 benchmarks_parallel: $(COMPILED_CAIRO0_PROGRAMS)
 	cargo bench -F parallel --bench criterion_prover
 	cargo bench -F parallel --bench criterion_verifier
 
+benchmarks_security: $(COMPILED_CAIRO0_PROGRAMS)
+	cargo bench -F parallel --bench criterion_prover_security
+	cargo bench -F parallel --bench criterion_verifier_security
+
+benchmarks_table: $(COMPILED_CAIRO0_PROGRAMS)
+	cargo bench -F parallel --bench criterion_table
+	cargo bench --bench dhat_prover
+
 benchmarks_parallel_all: $(COMPILED_CAIRO0_PROGRAMS)
 	cargo bench -F parallel
 

README.md

@@ -140,3 +140,138 @@ if you don´t have the tools for fuzzing installed use
 make fuzzer_tools
 ```
 
+## Benchmarks
+
+To get the results of the table below, run
+
+```
+make benchmarks_table
+```
+
+The results shown are from the execution of a Fibonacci program.
+
+Bits of security are used to ensure that the proof satisfies a certain conjuecturable security level.
+
+First table has the results that are independent of the hardware used.
+
+| n   | Trace length | Prover RAM | Proof size 80 bit security | Proof size 128 bit security |
+|-----|--------------|------------|----------------------------|-----------------------------|
+| 100 | 2^10         | 18.5 MB    | 270 KB                     | 476 KB                      |
+| 500 | 2^12         | 75.5 MB    | 335 KB                     | 591 KB                      |
+| 2k  | 2^14         | 302.1 MB   | 407 KB                     | 719 KB                      |
+| 5k  | 2^16         | 1.2 GB     | 488 KB                     | 862 KB                      |
+| 20k | 2^18         | 4.7 GB     | 576 KB                     | 1 MB                        |
+
+Second table has the results of the execution on an Apple M1 with 4 E and 4 P cores and 16 GB of RAM:
+
+<table>
+    <tr>
+        <th rowspan="2">Trace length</th>
+        <th rowspan="2">Trace time</th>
+        <th colspan="2" style="text-align:center">80 bit security</th>
+        <th colspan="2" style="text-align:center">128 bit security</th>
+    </tr>
+    <tr>
+        <th>Prover time</th>
+        <th>Verifier time</th>
+        <th>Prover time</th>
+        <th>Verifier time</th>
+    </tr>
+    <tr>
+        <td>2^10</td>
+        <td>0.9 ms</td>
+        <td>1.1 s</td>
+        <td>3.1 ms</td>
+        <td>1.1 s</td>
+        <td>4.7 ms</td>
+    </tr>
+    <tr>
+        <td>2^12</td>
+        <td>5.3 ms</td>
+        <td>335.5 ms</td>
+        <td>7.6 ms</td>
+        <td>336.4 ms</td>
+        <td>9.5 ms</td>
+    </tr>
+    <tr>
+        <td>2^14</td>
+        <td>24.7 ms</td>
+        <td>1.41 s</td>
+        <td>26.4 ms</td>
+        <td>1.42 s</td>
+        <td>29 ms</td>
+    </tr>
+    <tr>
+        <td>2^16</td>
+        <td>77.2 ms</td>
+        <td>5.8 s</td>
+        <td>108.8 ms</td>
+        <td>5.8 s</td>
+        <td>113.9 ms</td>
+    </tr>
+    <tr>
+        <td>2^18</td>
+        <td>312 ms</td>
+        <td>24.3 s</td>
+        <td>477.4 ms</td>
+        <td>24.3 s</td>
+        <td>481.7 ms</td>
+    </tr>
+</table>
+
+Third table has the results of the execution on an Intel Xeon Platinum with 4 cores and 16 GB of RAM:
+
+<table>
+     <tr>
+        <th rowspan="2">Trace length</th>
+        <th rowspan="2">Trace time</th>
+        <th colspan="2" style="text-align:center">80 bit security</th>
+        <th colspan="2" style="text-align:center">128 bit security</th>
+    </tr>
+    <tr>
+        <th>Prover time</th>
+        <th>Verifier time</th>
+        <th>Prover time</th>
+        <th>Verifier time</th>
+    </tr>
+    <tr>
+        <td>2^10</td>
+        <td>1.4 ms</td>
+        <td>2.5 s</td>
+        <td>6.3 ms</td>
+        <td>2.5 s</td>
+        <td>9.8 ms</td>
+    </tr>
+    <tr>
+        <td>2^12</td>
+        <td>8 ms</td>
+        <td>709 ms</td>
+        <td>13.3 ms</td>
+        <td>710.5 ms</td>
+        <td>17.7 ms</td>
+    </tr>
+    <tr>
+        <td>2^14</td>
+        <td>44.9 ms</td>
+        <td>3 s</td>
+        <td>41.1 ms</td>
+        <td>3 s</td>
+        <td>46.5 ms</td>
+    </tr>
+    <tr>
+        <td>2^16</td>
+        <td>140.8 s</td>
+        <td>12.2 s</td>
+        <td>160.6 ms</td>
+        <td>12.2 s</td>
+        <td>168 ms</td>
+    </tr>
+    <tr>
+        <td>2^18</td>
+        <td>700 ms</td>
+        <td>50.5 s</td>
+        <td>692.9 ms</td>
+        <td>50.5 s</td>
+        <td>702.6 ms</td>
+    </tr>
+</table>

benches/criterion_giza.rs

@@ -10,6 +10,8 @@ use lambdaworks_stark::{
     starks::proof::options::{self, SecurityLevel},
 };
 #[cfg(feature = "giza")]
+pub mod criterion_utils;
+#[cfg(feature = "giza")]
 pub mod functions;
 
 #[cfg(not(feature = "giza"))]

benches/criterion_prover.rs

@@ -1,14 +1,11 @@
 use criterion::{
-    black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
-};
-use lambdaworks_stark::{
-    cairo::{
-        air::generate_cairo_proof,
-        runner::run::{generate_prover_args, CairoVersion},
-    },
-    starks::proof::options::{ProofOptions, SecurityLevel},
+    criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
 };
+use criterion_utils::utils::run_cairo_bench_with_security_level;
+use functions::path::cairo0_program_path;
+use lambdaworks_stark::starks::proof::options::SecurityLevel;
 
+pub mod criterion_utils;
 pub mod functions;
 
 fn cairo_benches(c: &mut Criterion) {
@@ -39,25 +36,13 @@ fn cairo_benches(c: &mut Criterion) {
     );
 }
 
-fn cairo0_program_path(program_name: &str) -> String {
-    const CARGO_DIR: &str = env!("CARGO_MANIFEST_DIR");
-    const PROGRAM_BASE_REL_PATH: &str = "/cairo_programs/cairo0/";
-    let program_base_path = CARGO_DIR.to_string() + PROGRAM_BASE_REL_PATH;
-    program_base_path + program_name
-}
-
 fn run_cairo_bench(group: &mut BenchmarkGroup<'_, WallTime>, benchname: &str, program_path: &str) {
-    let program_content = std::fs::read(program_path).unwrap();
-    let proof_options = ProofOptions::new_secure(SecurityLevel::Provable80Bits, 3);
-    let (main_trace, pub_inputs) =
-        generate_prover_args(&program_content, &CairoVersion::V0, &None).unwrap();
-    println!("Generated main trace with {} rows", main_trace.n_rows());
-
-    group.bench_function(benchname, |bench| {
-        bench.iter(|| {
-            black_box(generate_cairo_proof(&main_trace, &pub_inputs, &proof_options).unwrap())
-        });
-    });
+    run_cairo_bench_with_security_level(
+        group,
+        benchname,
+        program_path,
+        SecurityLevel::Conjecturable80Bits,
+    );
 }
 
 criterion_group!(benches, cairo_benches);

benches/criterion_prover_70k.rs

@@ -1,15 +1,11 @@
 use criterion::{
-    black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
-    SamplingMode,
-};
-use lambdaworks_stark::{
-    cairo::{
-        air::generate_cairo_proof,
-        runner::run::{generate_prover_args, CairoVersion},
-    },
-    starks::proof::options::{ProofOptions, SecurityLevel},
+    criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion, SamplingMode,
 };
+use criterion_utils::utils::run_cairo_bench_with_security_level;
+use functions::path::cairo0_program_path;
+use lambdaworks_stark::starks::proof::options::SecurityLevel;
 
+pub mod criterion_utils;
 pub mod functions;
 
 fn fibo_70k_bench(c: &mut Criterion) {
@@ -36,24 +32,13 @@ fn fibo_70k_bench(c: &mut Criterion) {
     );
 }
 
-fn cairo0_program_path(program_name: &str) -> String {
-    const CARGO_DIR: &str = env!("CARGO_MANIFEST_DIR");
-    const PROGRAM_BASE_REL_PATH: &str = "/cairo_programs/cairo0/";
-    let program_base_path = CARGO_DIR.to_string() + PROGRAM_BASE_REL_PATH;
-    program_base_path + program_name
-}
-
 fn run_cairo_bench(group: &mut BenchmarkGroup<'_, WallTime>, benchname: &str, program_path: &str) {
-    let program_content = std::fs::read(program_path).unwrap();
-    let proof_options = ProofOptions::new_secure(SecurityLevel::Provable80Bits, 3);
-    let (main_trace, pub_inputs) =
-        generate_prover_args(&program_content, &CairoVersion::V0, &None).unwrap();
-
-    group.bench_function(benchname, |bench| {
-        bench.iter(|| {
-            black_box(generate_cairo_proof(&main_trace, &pub_inputs, &proof_options).unwrap())
-        });
-    });
+    run_cairo_bench_with_security_level(
+        group,
+        benchname,
+        program_path,
+        SecurityLevel::Provable128Bits,
+    );
 }
 
 criterion_group!(benches, fibo_70k_bench);