From 1bff23ed7a19cda711929a215be24352b7efc962 Mon Sep 17 00:00:00 2001
From: Abhishek Shah <abhishekshah3@gmail.com>
Date: Wed, 28 Feb 2024 23:03:13 +0530
Subject: [PATCH] Benchmark KvStore vs Sled backend (#4)

* Add <get> bench

* get bench group. work on set

* Benches for set/rm
---
 Cargo.lock           | 152 ++++++++++++++++++++++++++++++++++++++++---
 Cargo.toml           |   1 +
 benches/bench.rs     |  49 --------------
 lib/Cargo.toml       |   9 ++-
 lib/benches/bench.rs |  96 +++++++++++++++++++++++++++
 5 files changed, 247 insertions(+), 60 deletions(-)
 delete mode 100644 benches/bench.rs
 create mode 100644 lib/benches/bench.rs

diff --git a/Cargo.lock b/Cargo.lock
index 941429d..7afc2fa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11,6 +11,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstream"
 version = "0.6.12"
@@ -163,6 +169,33 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half 2.4.0",
+]
+
 [[package]]
 name = "clap"
 version = "2.34.0"
@@ -170,10 +203,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
 dependencies = [
  "bitflags 1.3.2",
- "textwrap",
+ "textwrap 0.11.0",
  "unicode-width",
 ]
 
+[[package]]
+name = "clap"
+version = "3.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
+dependencies = [
+ "bitflags 1.3.2",
+ "clap_lex 0.2.4",
+ "indexmap 1.9.3",
+ "textwrap 0.16.1",
+]
+
 [[package]]
 name = "clap"
 version = "4.4.6"
@@ -192,7 +237,7 @@ checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45"
 dependencies = [
  "anstream",
  "anstyle",
- "clap_lex",
+ "clap_lex 0.5.1",
  "strsim",
 ]
 
@@ -208,6 +253,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.5.1"
@@ -256,7 +310,7 @@ dependencies = [
  "atty",
  "cast",
  "clap 2.34.0",
- "criterion-plot",
+ "criterion-plot 0.4.5",
  "csv",
  "itertools",
  "lazy_static",
@@ -273,6 +327,32 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast",
+ "ciborium",
+ "clap 3.2.25",
+ "criterion-plot 0.5.0",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
 [[package]]
 name = "criterion-plot"
 version = "0.4.5"
@@ -283,6 +363,16 @@ dependencies = [
  "itertools",
 ]
 
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.8.3"
@@ -316,6 +406,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+
 [[package]]
 name = "csv"
 version = "1.3.0"
@@ -493,6 +589,22 @@ version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
 
+[[package]]
+name = "half"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "hashbrown"
 version = "0.14.3"
@@ -535,6 +647,16 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg 1.1.0",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.2.3"
@@ -542,7 +664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177"
 dependencies = [
  "equivalent",
- "hashbrown",
+ "hashbrown 0.14.3",
 ]
 
 [[package]]
@@ -595,13 +717,13 @@ version = "1.2.0"
 dependencies = [
  "assert_cmd",
  "clap 4.4.6",
- "criterion",
+ "criterion 0.4.0",
  "dotenv",
  "env_logger 0.10.0",
  "lazy_static",
  "log",
  "predicates",
- "rand 0.6.5",
+ "rand 0.8.5",
  "ron",
  "serde",
  "serde_json",
@@ -619,7 +741,7 @@ dependencies = [
  "assert_cmd",
  "clap 4.4.6",
  "common",
- "criterion",
+ "criterion 0.3.6",
  "env_logger 0.10.0",
  "kvs",
  "log",
@@ -727,6 +849,12 @@ version = "11.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
+[[package]]
+name = "os_str_bytes"
+version = "6.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -759,7 +887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap",
+ "indexmap 2.2.3",
 ]
 
 [[package]]
@@ -1186,7 +1314,7 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
 dependencies = [
- "half",
+ "half 1.8.2",
  "serde",
 ]
 
@@ -1288,6 +1416,12 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
+
 [[package]]
 name = "thiserror"
 version = "1.0.50"
diff --git a/Cargo.toml b/Cargo.toml
index 2ff7021..359ee25 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,3 +25,4 @@ serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0.104"
 thiserror = "1.0.38"
 prost = "0.12.3"
+
diff --git a/benches/bench.rs b/benches/bench.rs
deleted file mode 100644
index f834e0d..0000000
--- a/benches/bench.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-#![feature(test)]
-#![allow(unused)]
-
-extern crate test;
-use assert_cmd::prelude::*;
-use kvs::KvStore;
-use std::process::Command;
-use tempfile::TempDir;
-use test::{bench, Bencher};
-
-#[bench]
-fn bench_kvstore_open(b: &mut Bencher) {
-    let temp_dir = TempDir::new().unwrap();
-    b.iter(|| {
-        let mut store = test::black_box(KvStore::open(&temp_dir.path()).unwrap());
-    })
-}
-
-#[bench]
-fn bench_kvstore_set(b: &mut Bencher) {
-    let temp_dir = TempDir::new().unwrap();
-    let mut store = KvStore::open(&temp_dir.path()).unwrap();
-    b.iter(|| {
-        let key = test::black_box("key".to_string());
-        let value = test::black_box("value".to_string());
-        store.set(key, value);
-    })
-}
-
-#[bench]
-fn bench_kvstore_get(b: &mut Bencher) {
-    let temp_dir = TempDir::new().unwrap();
-    let mut store = KvStore::open(&temp_dir.path()).unwrap();
-    store.set("key".to_string(), "some_get_val".to_string());
-    b.iter(|| {
-        store.get("key".to_string());
-    })
-}
-
-
-#[bench]
-fn bench_kvstore_remove(b: &mut Bencher) {
-    let temp_dir = TempDir::new().unwrap();
-    let mut store = KvStore::open(&temp_dir.path()).unwrap();
-    store.set("key".to_string(), "some_get_val".to_string());
-    b.iter(|| {
-        store.remove("key".to_string());
-    })
-}
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index f04b418..837eb6d 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -23,8 +23,13 @@ sled = "0.34.7"
 
 [dev-dependencies]
 assert_cmd = "0.11"
-criterion = "0.3"
+criterion = { version = "0.4", features = ["html_reports"] }
 predicates = "1.0.0"
-rand = "0.6.5"
+rand = "0.8"
 tempfile = "3.0.7"
 walkdir = "2.2.7"
+
+[[bench]]
+name = "bench"
+# Disable rust-bench harness and prefer criterion's instead
+harness = false
diff --git a/lib/benches/bench.rs b/lib/benches/bench.rs
new file mode 100644
index 0000000..7678fc3
--- /dev/null
+++ b/lib/benches/bench.rs
@@ -0,0 +1,96 @@
+use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use kvs::{KvStore, KvsEngine, SledKvsEngine};
+use rand::Rng;
+use tempfile::TempDir;
+
+fn cold_start_get(c: &mut Criterion) {
+    let mut group = c.benchmark_group("GET");
+
+    group.bench_function("kvs: get key", |b: &mut Bencher<_>| {
+        // Setup for KVS
+        let temp_dir = TempDir::new().unwrap();
+        let mut store = KvStore::open(&temp_dir.path()).unwrap();
+        store
+            .set("key".to_string(), "some_get_val".to_string())
+            .unwrap();
+        b.iter(|| {
+            let _ = store.get("key".to_string()).unwrap();
+        })
+    });
+
+    group.bench_function("sled: get key", |b: &mut Bencher<_>| {
+        // Setup for SLED
+        let temp_dir = TempDir::new().unwrap();
+        let mut store = SledKvsEngine::open(&temp_dir.path()).unwrap();
+        store
+            .set("key".to_string(), "some_get_val".to_string())
+            .unwrap();
+        b.iter(|| {
+            let _ = store.get("key".to_string()).unwrap();
+        })
+    });
+    group.finish();
+}
+fn set_many_keys(c: &mut Criterion) {
+    let mut group = c.benchmark_group("SET/RM");
+    let temp_dir = TempDir::new().unwrap();
+    let mut store = KvStore::open(&temp_dir.path()).unwrap();
+    let test_data: Vec<(String, String)> = generate_test_data();
+    group.bench_function("kvs: set key", |b: &mut Bencher<_>| {
+        b.iter(|| {
+            for (k, v) in test_data.clone().into_iter() {
+                store.set(k, v).unwrap();
+            }
+        })
+    });
+    group.bench_function("kvs: remove keys", |b| {
+        b.iter(|| {
+            for (k, _) in test_data.clone().into_iter() {
+                store.remove(k).unwrap();
+            }
+        })
+    });
+    let temp_dir = TempDir::new().unwrap();
+    let mut store = SledKvsEngine::open(&temp_dir.path()).unwrap();
+    let test_data: Vec<(String, String)> = generate_test_data();
+    group.bench_function("sled: set key", |b: &mut Bencher<_>| {
+        b.iter(|| {
+            for (k, v) in test_data.clone().into_iter() {
+                store.set(k, v).unwrap();
+            }
+        })
+    });
+    group.bench_function("sled: remove keys", |b| {
+        b.iter(|| {
+            for (k, _) in test_data.clone().into_iter() {
+                store.remove(k).unwrap();
+            }
+        })
+    });
+    group.finish();
+}
+
+// Define a criterion group `kv_benches` with the benchmarks under it
+criterion_group!(kv_benches, cold_start_get, set_many_keys);
+// Run all benchmarks in a given group
+criterion_main!(kv_benches);
+
+fn generate_random_string(length: usize) -> String {
+    let mut rng = rand::thread_rng();
+    let bytes = (0..length)
+        .map(|_| rng.sample(rand::distributions::Alphanumeric))
+        .collect::<Vec<u8>>();
+
+    String::from_utf8(bytes).unwrap()
+}
+
+fn generate_test_data() -> Vec<(String, String)> {
+    let mut data: Vec<(String, String)> = vec![];
+
+    for item in &mut data {
+        item.0 = generate_random_string(40); // Adjust the length as needed
+        item.1 = generate_random_string(100); // Adjust the length as needed
+    }
+
+    data
+}