diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d54fa2f..b1404ce 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -15,8 +15,27 @@ jobs: runs-on: ubuntu-latest steps: + - name: Free up space on runner + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: Install protoc + run: | + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Rust Cache + uses: Swatinem/rust-cache@v2.5.0 + with: + cache-on-failure: true + cache-all-crates: true + - uses: actions/checkout@v3 - name: Build - run: cargo build --verbose + run: cargo build --workspace --verbose + - name: Run tests run: cargo test --verbose diff --git a/.gitignore b/.gitignore index eb5a316..d2670c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ target +*.log +*.index \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..bc3dded --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "cppvsdbg", + "request": "launch", + "name": "Debug cmd in executable 'kvs'", + "program": "${workspaceFolder}/target/debug/kvs.exe", + "args": [ + "set", + "horseradish", + "clams" + ], + "cwd": "${workspaceFolder}", + "preLaunchTask": "rust: cargo build", + }, + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..4d8289d --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,21 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "rust: cargo build", + "type": "cargo", + "command": "build", + "args": [ + // "--package kvs", + // "--bin kvs" + ], + "options": { + "cwd": "${workspaceFolder}", + }, + "problemMatcher": [ + "$rustc" + ], + "group": "build", + } + ] +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index eb073c6..941429d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540" dependencies = [ "anstyle", "anstyle-parse", @@ -27,9 +27,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -46,7 +46,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -56,9 +56,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] +[[package]] +name = "anyhow" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" + [[package]] name = "assert_cmd" version = "0.11.1" @@ -71,6 +77,26 @@ dependencies = [ "predicates-tree", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -98,6 +124,30 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.83" @@ -113,6 +163,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags 1.3.2", + "textwrap", + "unicode-width", +] + [[package]] name = "clap" version = "4.4.6" @@ -153,12 +214,129 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "common" +version = "1.2.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap 2.34.0", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg 1.1.0", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "difference" version = "2.0.0" @@ -171,6 +349,22 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -184,6 +378,25 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.4" @@ -192,7 +405,7 @@ checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -223,159 +436,655 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "float-cmp" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "indexmap" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi 0.3.3", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kvs" +version = "1.2.0" +dependencies = [ + "assert_cmd", + "clap 4.4.6", + "criterion", + "dotenv", + "env_logger 0.10.0", + "lazy_static", + "log", + "predicates", + "rand 0.6.5", + "ron", + "serde", + "serde_json", + "sled", + "tempfile", + "thiserror", + "walkdir", +] + +[[package]] +name = "kvs-client" +version = "1.2.0" +dependencies = [ + "anyhow", + "assert_cmd", + "clap 4.4.6", + "common", + "criterion", + "env_logger 0.10.0", + "kvs", + "log", + "predicates", + "prost", + "rand 0.6.5", + "tempfile", + "walkdir", +] + +[[package]] +name = "kvs-server" +version = "1.2.0" +dependencies = [ + "anyhow", + "clap 4.4.6", + "common", + "env_logger 0.11.2", + "kvs", + "log", + "prost", + "tracing", + "uuid", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" + +[[package]] +name = "linux-raw-sys" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg 1.1.0", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "num-traits", + "proc-macro2", ] [[package]] -name = "heck" -version = "0.4.1" +name = "rand" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] [[package]] -name = "hermit-abi" -version = "0.3.3" +name = "rand" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] [[package]] -name = "humantime" -version = "2.1.0" +name = "rand_chacha" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", +] [[package]] -name = "is-terminal" -version = "0.4.9" +name = "rand_chacha" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ - "hermit-abi", - "rustix", - "windows-sys", + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] -name = "itoa" -version = "1.0.9" +name = "rand_core" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" - -[[package]] -name = "kvs" -version = "0.1.0" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" dependencies = [ - "assert_cmd", - "clap", - "dotenv", - "env_logger", - "lazy_static", - "log", - "predicates", - "ron", - "serde", - "serde_json", - "tempfile", - "thiserror", - "walkdir", + "rand_core 0.4.2", ] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "rand_core" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" [[package]] -name = "libc" -version = "0.2.149" +name = "rand_core" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] [[package]] -name = "linux-raw-sys" -version = "0.4.8" +name = "rand_hc" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] [[package]] -name = "log" -version = "0.4.20" +name = "rand_isaac" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] [[package]] -name = "memchr" -version = "2.6.4" +name = "rand_jitter" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] [[package]] -name = "normalize-line-endings" -version = "0.3.0" +name = "rand_os" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] [[package]] -name = "num-traits" -version = "0.2.17" +name = "rand_pcg" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "autocfg", + "autocfg 0.1.8", + "rand_core 0.4.2", ] [[package]] -name = "predicates" -version = "1.0.8" +name = "rand_xorshift" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" dependencies = [ - "difference", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", + "rand_core 0.3.1", ] [[package]] -name = "predicates-core" -version = "1.0.6" +name = "rayon" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] [[package]] -name = "predicates-tree" -version = "1.0.9" +name = "rayon-core" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "predicates-core", - "termtree", + "crossbeam-deque", + "crossbeam-utils", ] [[package]] -name = "proc-macro2" -version = "1.0.68" +name = "rdrand" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" dependencies = [ - "unicode-ident", + "rand_core 0.3.1", ] [[package]] -name = "quote" -version = "1.0.33" +name = "redox_syscall" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "proc-macro2", + "bitflags 1.3.2", ] [[package]] @@ -438,7 +1147,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -456,6 +1165,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.188" @@ -465,6 +1180,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.188" @@ -487,6 +1212,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + [[package]] name = "strsim" version = "0.10.0" @@ -512,9 +1259,9 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -532,38 +1279,105 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +dependencies = [ + "getrandom", + "rand 0.8.5", +] + [[package]] name = "walkdir" version = "2.4.0" @@ -574,6 +1388,88 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" @@ -611,7 +1507,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.3", ] [[package]] @@ -620,13 +1525,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +dependencies = [ + "windows_aarch64_gnullvm 0.52.3", + "windows_aarch64_msvc 0.52.3", + "windows_i686_gnu 0.52.3", + "windows_i686_msvc 0.52.3", + "windows_x86_64_gnu 0.52.3", + "windows_x86_64_gnullvm 0.52.3", + "windows_x86_64_msvc 0.52.3", ] [[package]] @@ -635,38 +1555,80 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" diff --git a/Cargo.toml b/Cargo.toml index 1e47101..2ff7021 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,33 +1,27 @@ -[package] -name = "kvs" -version = "0.1.0" -edition = "2021" -description = "The cargo project, kvs, builds a command-line key-value store client called kvs, which in turn calls into a library called kvs." +[workspace.package] authors = ["Abhishek Shah "] +description = "A key-value store called kvs" +edition = "2021" +version = "1.2.0" +license = "MIT" +# https://doc.rust-lang.org/cargo/reference/workspaces.html -[[bin]] -name = "kvs" -path = "src/bin/main.rs" +[workspace] +resolver = "2" -[lib] -name = "kvs" -path = "src/lib.rs" -doctest = false -test = false +members = ["crates/common", "crates/kvs-client", "crates/kvs-server", "lib"] +default-members = ["lib", "crates/kvs-client", "crates/kvs-server"] -[dependencies] +[workspace.dependencies] +anyhow = "1.0.80" clap = { version = "4.1.6", features = ["derive"] } dotenv = "0.15.0" env_logger = "0.10.0" +kvs = { path = "lib" } lazy_static = "1.4.0" log = "0.4.20" ron = "0.8.1" serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.104" thiserror = "1.0.38" - -[dev-dependencies] -assert_cmd = "0.11.0" -predicates = "1.0.0" -tempfile = "3.4.0" -walkdir = "2.3.2" +prost = "0.12.3" diff --git a/benches/bench.rs b/benches/bench.rs new file mode 100644 index 0000000..f834e0d --- /dev/null +++ b/benches/bench.rs @@ -0,0 +1,49 @@ +#![feature(test)] +#![allow(unused)] + +extern crate test; +use assert_cmd::prelude::*; +use kvs::KvStore; +use std::process::Command; +use tempfile::TempDir; +use test::{bench, Bencher}; + +#[bench] +fn bench_kvstore_open(b: &mut Bencher) { + let temp_dir = TempDir::new().unwrap(); + b.iter(|| { + let mut store = test::black_box(KvStore::open(&temp_dir.path()).unwrap()); + }) +} + +#[bench] +fn bench_kvstore_set(b: &mut Bencher) { + let temp_dir = TempDir::new().unwrap(); + let mut store = KvStore::open(&temp_dir.path()).unwrap(); + b.iter(|| { + let key = test::black_box("key".to_string()); + let value = test::black_box("value".to_string()); + store.set(key, value); + }) +} + +#[bench] +fn bench_kvstore_get(b: &mut Bencher) { + let temp_dir = TempDir::new().unwrap(); + let mut store = KvStore::open(&temp_dir.path()).unwrap(); + store.set("key".to_string(), "some_get_val".to_string()); + b.iter(|| { + store.get("key".to_string()); + }) +} + + +#[bench] +fn bench_kvstore_remove(b: &mut Bencher) { + let temp_dir = TempDir::new().unwrap(); + let mut store = KvStore::open(&temp_dir.path()).unwrap(); + store.set("key".to_string(), "some_get_val".to_string()); + b.iter(|| { + store.remove("key".to_string()); + }) +} diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..7116b7e --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# build.sh + +cargo build --release --workspace diff --git a/client.sh b/client.sh new file mode 100755 index 0000000..ad77cfe --- /dev/null +++ b/client.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# client.sh +RUST_LOG=info cargo r -p kvs-client "$@" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml new file mode 100644 index 0000000..8835cca --- /dev/null +++ b/crates/common/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "common" +authors.workspace = true +description.workspace = true +edition.workspace = true +version.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +prost = { workspace = true } +prost-types = "0.12.3" + +[build-dependencies] +prost-build = "0.12.3" diff --git a/crates/common/build.rs b/crates/common/build.rs new file mode 100644 index 0000000..91db38a --- /dev/null +++ b/crates/common/build.rs @@ -0,0 +1,7 @@ +extern crate prost_build; + +fn main() { + let mut config = prost_build::Config::new(); + config.protoc_arg("--experimental_allow_proto3_optional"); + config.compile_protos(&["src/message.proto"], &["src/"]).unwrap(); +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs new file mode 100644 index 0000000..cc1f82c --- /dev/null +++ b/crates/common/src/lib.rs @@ -0,0 +1,8 @@ +//! Protobuf definitions for over the wire communication +//! Types generated via the build script and prost/protoc. +//! Running build on this crate will place these generated types under target//common-/out/ + +pub mod msg { + include!(concat!(env!("OUT_DIR"), "/kvs_message.rs")); +} +pub use msg::*; diff --git a/crates/common/src/message.proto b/crates/common/src/message.proto new file mode 100644 index 0000000..b88426c --- /dev/null +++ b/crates/common/src/message.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +package kvs_message; + +// Request message types for different operations +enum MessageType { + SET = 0; + GET = 1; + RM = 2; +} + +// Message to set a key-value pair +message Set { + string key = 1; + string value = 2; +} + +// Message to get a value for a key +message Get { + string key = 1; +} + +// Message to remove a key-value pair +message Rm { + string key = 1; +} + +// Message containing data for different operations +message Message { + MessageType type = 1; + oneof payload { + Set set = 2; + Get get = 3; + Rm rm = 4; + } +} + +// Response from Server to Client if any +message Response { + bool success = 1; + // Contains error message if success is false + optional string value = 2; +} diff --git a/crates/kvs-client/Cargo.toml b/crates/kvs-client/Cargo.toml new file mode 100644 index 0000000..071da9f --- /dev/null +++ b/crates/kvs-client/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "kvs-client" +version = { workspace = true } +edition = "2021" +authors = { workspace = true } +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "kvs-client" +path = "src/client.rs" + + +[dependencies] +kvs = { workspace = true } +clap = { workspace = true } +log = { workspace = true } +env_logger = { workspace = true } +anyhow = "1.0.80" +common = { path = "../common" } +prost = { workspace = true } + +[dev-dependencies] +assert_cmd = "0.11" +criterion = "0.3" +predicates = "1.0.0" +rand = "0.6.5" +tempfile = "3.0.7" +walkdir = "2.2.7" diff --git a/crates/kvs-client/src/client.rs b/crates/kvs-client/src/client.rs new file mode 100644 index 0000000..461c068 --- /dev/null +++ b/crates/kvs-client/src/client.rs @@ -0,0 +1,103 @@ +use anyhow::Context; +use common::message::Payload; +use common::{Get, Message, MessageType, Response, Rm, Set}; +use kvs::cli::{Action, GetCmd, RmCmd, SetCmd}; +use kvs::exit_program; +use log::trace; +use prost::Message as ProstMessage; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; + +fn main() -> anyhow::Result<()> { + ::env_logger::init(); + let cli = ::parse(); + let server = cli.addr.parse::()?; + let mut server = TcpStream::connect(server)?; + log::info!("🌐 Connected to server [{}]", server.peer_addr()?); + if match cli.action { + Action::Set(SetCmd { key, value }) => { + log::debug!("✉️ Requesting -> Set {} = {}", key, value); + send(Payload::Set(Set { key, value }), &mut server) + } + Action::Get(GetCmd { key }) => { + log::debug!("✉️ Requesting -> Get {}", key); + send(Payload::Get(Get { key }), &mut server) + } + Action::Remove(RmCmd { key }) => { + log::debug!("✉️ Requesting -> Rm {}", key); + send(Payload::Rm(Rm { key }), &mut server) + } + } + .is_err() + { + exit_program(1); + } + exit_program(0); +} + +fn send(payload: Payload, server: &mut TcpStream) -> anyhow::Result<()> { + let mut message_bytes: Vec = vec![]; + let r#type = match payload { + Payload::Set { .. } => MessageType::Set as i32, // 0 + Payload::Get { .. } => MessageType::Get as i32, // 1 + Payload::Rm { .. } => MessageType::Rm as i32, // 2 + }; + let message = Message { + r#type, + payload: Some(payload), + }; + trace!("Message request -> {:#?}", message); + { + // Send message towards server + message + .encode(&mut message_bytes) + .context("failed to encode message into bytes")?; + server.write_all(&message_bytes)?; + server.flush()?; + log::debug!("Written {} bytes to server stream", message_bytes.len()); + log::trace!("Bytes -> {message_bytes:?}"); + server.shutdown(std::net::Shutdown::Write)?; + } + // Clear buffer, await response + message_bytes.clear(); + { + // We depend on the server to shutdown the stream after it's finished sending a response + let bytes_read = server.read_to_end(&mut message_bytes)?; + log::debug!("Got {} bytes back ", bytes_read); + let response = Response::decode(&message_bytes[0..bytes_read]) + .context("failed to decode message response from server")?; + if response.success { + if let Some(v) = response.value { + println!("{}", v); + } else { + // If we fetch an unset key, we can print + if r#type == MessageType::Get as i32 { + eprintln!("Key not found"); + } + } + // println!( + // "✅ {} Success", + // match MessageType::try_from(r#type).expect("message type is valid") { + // MessageType::Get => "GET", + // MessageType::Set => "SET", + // MessageType::Rm => "RM", + // } + // ); + } else { + if let Some(err) = response.value { + eprintln!("❌ Server Error: {err}"); + } + } + } + Ok(()) +} + +#[derive(Debug, clap::Parser)] +#[command(version)] +struct Cli { + #[clap(subcommand)] + action: Action, + /// Server location + #[arg(short, long, default_value = "127.0.0.1:4000")] + addr: String, +} diff --git a/crates/kvs-client/tests/cli.rs b/crates/kvs-client/tests/cli.rs new file mode 100644 index 0000000..2897a49 --- /dev/null +++ b/crates/kvs-client/tests/cli.rs @@ -0,0 +1,352 @@ +use assert_cmd::prelude::*; +use predicates::str::contains; +use std::fs::{self, File}; +use std::process::Command; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; +use tempfile::TempDir; + +// `kvs-client` with no args should exit with a non-zero code. +#[test] +fn client_cli_no_args() { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-client").unwrap(); + cmd.current_dir(&temp_dir).assert().failure(); +} + +#[test] +fn client_cli_invalid_get() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "extra", "field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", "invalid-addr", "get", "key"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--unknown-flag", "get", "key"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +#[test] +fn client_cli_invalid_set() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "missing_field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "key", "value", "extra_field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", "invalid-addr", "set", "key", "value"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key", "--unknown-flag"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--unknown-flag", "get", "key"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +#[test] +fn client_cli_invalid_rm() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "extra", "field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", "invalid-addr", "rm", "key"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--unknown-flag", "rm", "key"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "key", "--unknown-flag"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +#[test] +fn client_cli_invalid_subcommand() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["unknown"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +// `kvs-client -V` should print the version +#[test] +fn client_cli_version() { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-client").unwrap(); + cmd.args(&["-V"]) + .current_dir(&temp_dir) + .assert() + .stdout(contains(env!("CARGO_PKG_VERSION"))); +} + +// `kvs-server -V` should print the version +#[test] +fn server_cli_version() { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + cmd.args(&["-V"]) + .current_dir(&temp_dir) + .assert() + .stdout(contains(env!("CARGO_PKG_VERSION"))); +} + +#[test] +fn cli_log_configuration() { + let temp_dir = TempDir::new().unwrap(); + let stderr_path = temp_dir.path().join("stderr"); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + let mut child = cmd + .args(&["--engine", "kvs", "--addr", "127.0.0.1:4001"]) + .current_dir(&temp_dir) + .stderr(File::create(&stderr_path).unwrap()) + .spawn() + .unwrap(); + thread::sleep(Duration::from_secs(1)); + child.kill().expect("server exited before killed"); + + let content = fs::read_to_string(&stderr_path).expect("unable to read from stderr file"); + assert!(content.contains(env!("CARGO_PKG_VERSION"))); + assert!(content.contains("kvs")); + assert!(content.contains("127.0.0.1:4001")); +} +#[test] +fn cli_wrong_engine() { + // sled first, kvs second + { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + let mut child = cmd + .args(&["--engine", "sled", "--addr", "127.0.0.1:4002"]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + thread::sleep(Duration::from_secs(1)); + child.kill().expect("server exited before killed"); + + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + cmd.args(&["--engine", "kvs", "--addr", "127.0.0.1:4003"]) + .current_dir(&temp_dir) + .assert() + .failure(); + } + + // kvs first, sled second + { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + let mut child = cmd + .args(&["--engine", "kvs", "--addr", "127.0.0.1:4002"]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + thread::sleep(Duration::from_secs(1)); + child.kill().expect("server exited before killed"); + + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + cmd.args(&["--engine", "sled", "--addr", "127.0.0.1:4003"]) + .current_dir(&temp_dir) + .assert() + .failure(); + } +} + +fn cli_access_server(engine: &str, addr: &str) { + let (sender, receiver) = mpsc::sync_channel(0); + let temp_dir = TempDir::new().unwrap(); + let mut server = Command::cargo_bin("kvs-server").unwrap(); + let mut child = server + .args(&["--engine", engine, "--addr", addr]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + let handle = thread::spawn(move || { + let _ = receiver.recv(); // wait for main thread to finish + child.kill().expect("server exited before killed"); + }); + thread::sleep(Duration::from_secs(1)); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "set", "key1", "value1"]) + .current_dir(&temp_dir) + .assert() + .success(); + // .stdout(is_empty()); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "get", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout("value1\n"); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "set", "key1", "value2"]) + .current_dir(&temp_dir) + .assert() + .success(); + // .stdout(is_empty()); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "get", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout("value2\n"); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "get", "key2"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(contains("Key not found")); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "rm", "key2"]) + .current_dir(&temp_dir) + .assert() + .success() + .stderr(contains("Key not found")); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "set", "key2", "value3"]) + .current_dir(&temp_dir) + .assert() + .success(); + // .stdout(is_empty()); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "rm", "key1"]) + .current_dir(&temp_dir) + .assert() + .success(); + // .stdout(is_empty()); + + sender.send(()).unwrap(); + handle.join().unwrap(); + + // Reopen and check value + let (sender, receiver) = mpsc::sync_channel(0); + let mut server = Command::cargo_bin("kvs-server").unwrap(); + let mut child = server + .args(&["--engine", engine, "--addr", addr]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + let handle = thread::spawn(move || { + let _ = receiver.recv(); // wait for main thread to finish + child.kill().expect("server exited before killed"); + }); + thread::sleep(Duration::from_secs(1)); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "get", "key2"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(contains("value3")); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["--addr", addr, "get", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(contains("Key not found")); + sender.send(()).unwrap(); + handle.join().unwrap(); +} +#[test] +fn cli_access_server_kvs_engine() { + cli_access_server("kvs", "127.0.0.1:4004"); +} + +#[test] +#[ignore = r#"Error: IO error: could not acquire lock on '/tmp/.tmp1srY2h/db': +Os { code: 11, kind: WouldBlock, message: "Resource temporarily unavailable" } +Test works manually"#] +fn cli_access_server_sled_engine() { + cli_access_server("sled", "127.0.0.1:4005"); +} diff --git a/crates/kvs-server/Cargo.toml b/crates/kvs-server/Cargo.toml new file mode 100644 index 0000000..20eea01 --- /dev/null +++ b/crates/kvs-server/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "kvs-server" +version = { workspace = true } +edition = "2021" +authors = { workspace = true } + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "kvs-server" +path = "src/server.rs" + +[dependencies] +anyhow = { workspace = true } +log = { workspace = true } +kvs = { workspace = true } +clap = { workspace = true } +tracing = { version = "0.1.40", features = ["log"] } +# todo: Switch to tracing-subscriber and remove env_logger/log +# tracing-subscriber = "0.3.18" +env_logger = "0.11.2" +uuid = { version = "1.7.0", features = ["v4", "fast-rng"] } +common = { path = "../common" } +prost = { workspace = true } diff --git a/crates/kvs-server/notes.md b/crates/kvs-server/notes.md new file mode 100644 index 0000000..25f66ad --- /dev/null +++ b/crates/kvs-server/notes.md @@ -0,0 +1,28 @@ +## Reading from TcpStream + +In this server we make use of `[0u8; 1024]` as the buffer capacity but that can easily be extended +You can use [BufRead](https://doc.rust-lang.org/std/io/trait.BufRead.html) or you can rely on the following code : + +```rust +use std::io::{Read, Result}; + +// ... other code ... + +let mut buffer = Vec::new(); + +loop { + // Read into a temporary buffer of 1024 bytes + let mut temp_buffer = [0_u8; 1024]; + let bytes_read = stream.read(&mut temp_buffer)?; + + // Append the read bytes to the main buffer + buffer.extend_from_slice(&temp_buffer[..bytes_read]); + + // Break if we've reached the end of the stream + if bytes_read == 0 { + break; + } +} + +trace!("{bytes_read} bytes read: {:?}", &buffer); +``` \ No newline at end of file diff --git a/crates/kvs-server/src/request.rs b/crates/kvs-server/src/request.rs new file mode 100644 index 0000000..1ce2fb8 --- /dev/null +++ b/crates/kvs-server/src/request.rs @@ -0,0 +1,114 @@ +use crate::Backend; +use anyhow::{anyhow, bail, Context}; +use common::{message::Payload, Get, Message, Response, Rm, Set}; +use kvs::DbError; +use prost::Message as ProstMessage; +use std::{ + io::{Read, Write}, + net::{Shutdown, TcpStream}, +}; +#[allow(unused_imports)] +use tracing::{debug, error, info, trace}; + +pub(crate) fn serve_request(backend: &mut Backend, mut stream: TcpStream) -> anyhow::Result<()> { + // Note: If you're using `read_to_end`, you can simply use a 0-length `vec![]` that will be coerced to the + // correct length during read. Say N bytes are read, so the vec will be N bytes long + // In this case however, we write our code using 1024 bytes and use the `bytes_read` a crucial variable + // to slice into the buffer for correct decoding of the protobuf. + let mut buffer: Vec = vec![0_u8; 1024]; + let bytes_read = stream.read(&mut buffer)?; + trace!("{bytes_read} bytes read : {:?}", &buffer[..bytes_read]); + if buffer.iter().all(|x| *x == 0) { + bail!("Request is zeroes 0️.. aborting"); + } + { + /* Response */ + let response: Vec = handle_request(backend, &buffer[..bytes_read])?; + drop(buffer); + stream.write_all(response.as_slice())?; + stream.flush()?; + stream.shutdown(Shutdown::Write)?; + trace!("Request completed 🚀"); + } + Ok(()) +} +// This functions returns a Result, whose Err variant is supposed to notify our server +// That some processing has failed. Kvs Backend errors are handled differently in that +// the failure is logged, and the client is notified with a Response { success: false } +fn handle_request(backend: &mut Backend, buffer: &[u8]) -> anyhow::Result> { + trace!("🔄 Processing request"); + // Note, the type of request is embedded both in the `type` and `payload` fields of `Message` + let request: Message = Message::decode(buffer).with_context(|| { + error!("🚨 Failed to parse request from client",); + "🚨 Server cannot decode request" + })?; + let payload = request + .payload + .ok_or(anyhow!("🚨 Missing payload in Request"))?; + // No matter error or success, we create a response to send back to the client + let response = match payload { + Payload::Set(Set { key, value }) => { + trace!("🔄 Processing Set {key}->{value} request"); + match backend.set(key, value) { + Ok(()) => Response { + success: true, + value: None, + }, + // A backend Err indicates that our KVS failed but we must also notify + // this to the client. We follow this logic with all other arms + Err(e) => { + error!("🚨 Backend failed to SET key-value pair: {}", e); + Response { + success: false, + value: None, + } + } + } + } + Payload::Get(Get { key }) => { + trace!("🔄 Processing Get {key} request"); + match backend.get(key) { + Ok(Some(value)) => Response { + success: true, + value: Some(value), + }, + Ok(None) => Response { + success: true, + value: Some(format!("Key not found")), + }, + Err(e) => { + error!("🚨 Backend failed to GET key-value pair: {}", e); + Response { + success: false, + value: None, + } + } + } + } + Payload::Rm(Rm { key }) => { + trace!("🔄 Processing Remove {key} request"); + match backend.remove(key) { + Ok(()) => Response { + success: true, + value: None, + }, + Err(e) => { + error!("🚨 Backend failed to RM key-value pair: {}", e); + Response { + success: false, + // TODO: How can we match on e if DbError doesn't implement PartialEq? + value: match e { + DbError::KeyNotFound => Some(format!("Key not found")), + _ => None, + }, + } + } + } + } + }; + let mut buffer: Vec = vec![]; + response + .encode(&mut buffer) + .context("Server failed to encode response back to client")?; + Ok(buffer) +} diff --git a/crates/kvs-server/src/server.rs b/crates/kvs-server/src/server.rs new file mode 100644 index 0000000..903d055 --- /dev/null +++ b/crates/kvs-server/src/server.rs @@ -0,0 +1,141 @@ +use anyhow::bail; +use env_logger::{Builder, Target}; +use kvs::{exit_program, KvStore, SledKvsEngine}; +use request::serve_request; +use std::env; +use std::ffi::OsString; +use std::net::{SocketAddr, TcpListener}; +use std::path::PathBuf; +use tracing::{error, info}; +mod request; +#[tracing::instrument] +fn main() -> anyhow::Result<()> { + Builder::new() + .target(Target::Stderr) + .filter_level(log::LevelFilter::Info) + .init(); + let KvsServer { socket, engine, .. } = ::parse(); + let socket: SocketAddr = socket.parse().expect("Failed to parse socket address"); + let engine_str = engine.expect("clap default used"); + let existing_db = match check_db(env::current_dir()?) { + Ok(db) => db, + Err(err) => { + error!("{}", err); + exit_program(4); + } + }; + let mut backend: Backend = match engine_str.to_lowercase().as_str() { + "kvs" => { + if existing_db == Db::Sled { + exit_program(10); + }; + Backend::Kvs(KvStore::open(env::current_dir()?)?) + } + "sled" => { + if existing_db == Db::Kvs { + exit_program(11); + }; + Backend::Sled(SledKvsEngine::open(env::current_dir()?)?) + } + _ => { + error!("Unsupported Engine"); + exit_program(2); + } + }; + info!("Starting KVS server version {}", env!("CARGO_PKG_VERSION")); + info!( + "Server configuration - IP:PORT: {socket}, Storage Engine: {}", + engine_str + ); + + let server = TcpListener::bind(socket).expect("Failed to bind to socket"); + for stream in server.incoming() { + let request_id = uuid::Uuid::new_v4(); + let span = tracing::info_span!("Request Processing", %request_id); + let _span_enter = span.enter(); + if let Err(err) = serve_request(&mut backend, stream?) { + error!(%err) + } + } + Ok(()) +} + +#[derive(clap::Parser)] +#[command(version)] +struct KvsServer { + #[arg(long = "addr", short = 'a', default_value = "127.0.0.1:4000")] + // Socket v4 or v6 -> IP:PORT + socket: String, + #[arg(long, short, default_value = "kvs")] + /// KV backend to use. + engine: Option, +} +enum Backend { + Kvs(KvStore), + Sled(SledKvsEngine), +} + +impl std::fmt::Display for Backend { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Backend::Kvs(_) => write!(f, "Kvs"), + Backend::Sled(_) => write!(f, "Sled"), + } + } +} + +impl std::ops::Deref for Backend { + type Target = dyn kvs::KvsEngine; + fn deref(&self) -> &Self::Target { + match self { + Backend::Kvs(kvs) => kvs, + Backend::Sled(sled) => sled, + } + } +} +impl std::ops::DerefMut for Backend { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + Backend::Kvs(kvs) => kvs, + Backend::Sled(sled) => sled, + } + } +} +#[derive(Debug, Default, PartialEq)] +enum Db { + Sled, + Kvs, + #[default] + None, +} +fn check_db(dir: PathBuf) -> anyhow::Result { + if !dir.exists() { + bail!("Directory does not exist"); + } + if !dir.is_dir() { + bail!("Path is not a directory"); + } + let mut sled_db = false; + let mut kvs_db = false; + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries { + if let Ok(entry) = entry { + if entry.file_name() == OsString::from("db") { + println!("Sled found"); + sled_db = true; + } else if entry.file_name().to_str().unwrap().starts_with("kv_") + && entry.file_name().to_str().unwrap().ends_with(".log") + { + println!("Kvs found"); + kvs_db = true; + } + } + } + } + match (sled_db, kvs_db) { + (true, true) => bail!("Both databases found. Abort"), + (true, false) => Ok(Db::Sled), + (false, true) => Ok(Db::Kvs), + (false, false) => Ok(Db::None), + } +} diff --git a/lib/Cargo.toml b/lib/Cargo.toml new file mode 100644 index 0000000..f04b418 --- /dev/null +++ b/lib/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "kvs" +authors.workspace = true +description.workspace = true +edition = { workspace = true } +version = { workspace = true } + +[[bin]] +name = "kvs" +path = "src/bin/main.rs" + +[dependencies] +clap = { workspace = true } +dotenv = { workspace = true } +env_logger = { workspace = true } +lazy_static = { workspace = true } +log = { workspace = true } +ron = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +sled = "0.34.7" + +[dev-dependencies] +assert_cmd = "0.11" +criterion = "0.3" +predicates = "1.0.0" +rand = "0.6.5" +tempfile = "3.0.7" +walkdir = "2.2.7" diff --git a/src/bin/main.rs b/lib/src/bin/main.rs similarity index 93% rename from src/bin/main.rs rename to lib/src/bin/main.rs index f00cc87..941095d 100644 --- a/src/bin/main.rs +++ b/lib/src/bin/main.rs @@ -1,5 +1,5 @@ //! This builds the `kvs` executable -use kvs::cli; +use kvs::{cli, exit_program, KvsEngine}; use log::{error, info}; use std::env; fn main() -> kvs::Result<()> { @@ -57,7 +57,3 @@ fn main() -> kvs::Result<()> { unreachable!("Action (subcommands) are required"); } } -/// Non-zero exit code indicates a program error -fn exit_program(code: i32) -> ! { - std::process::exit(code) -} diff --git a/src/cli.rs b/lib/src/cli.rs similarity index 100% rename from src/cli.rs rename to lib/src/cli.rs diff --git a/src/error.rs b/lib/src/error.rs similarity index 81% rename from src/error.rs rename to lib/src/error.rs index 0e8a59e..223c183 100644 --- a/src/error.rs +++ b/lib/src/error.rs @@ -5,7 +5,7 @@ use std::io; use crate::cli::Action; -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug,)] /// Database Error pub enum DbError { /// KvStore accessed before initialization on disk @@ -32,6 +32,12 @@ pub enum DbError { /// Ron SpannedResult error #[error("{}", _0)] RonSpanned(#[from] ron::error::SpannedError), + /// Sled Error + #[error("{}", _0)] + SledError(#[from] sled::Error), + /// Sled byte UTF-8 cast failure + #[error("{}", _0)] + SledUtf8Error(#[from] std::string::FromUtf8Error), } /// KvStore Result type, with error variant representing Database errors diff --git a/src/lib.rs b/lib/src/lib.rs similarity index 89% rename from src/lib.rs rename to lib/src/lib.rs index 16c6c58..67ca62f 100644 --- a/src/lib.rs +++ b/lib/src/lib.rs @@ -40,7 +40,9 @@ use std::{ pub mod cli; mod error; +mod utils; pub use error::{DbError, Result}; +pub use utils::*; use crate::cli::{Action, RmCmd, SetCmd}; @@ -51,6 +53,16 @@ lazy_static! { .separate_tuple_members(false); } +/// Backend for KvStore +pub trait KvsEngine { + /// Set key to value + fn set(&mut self, key: String, value: String) -> Result<()>; + /// Query for key + fn get(&self, key: String) -> Result>; + /// Remove key + fn remove(&mut self, key: String) -> Result<()>; +} + /// File offset pub type Offset = u64; /// KvStore implementation @@ -248,10 +260,10 @@ fn read_action_from_log(disk: &mut File) -> Result> { Ok(log) } -impl KvStore { +impl KvsEngine for KvStore { /// Set : When setting a key to a value, kvs writes the set command to disk in a sequential log, /// then stores the log pointer (file offset) of that command in the in-memory index from key to pointer. - pub fn set(&mut self, key: String, value: String) -> Result<()> { + fn set(&mut self, key: String, value: String) -> Result<()> { if self.map.len() > 500 { // trigger compaction (*self).compaction()?; @@ -273,13 +285,13 @@ impl KvStore { // write serialized to self.disk // TODO : Maybe think about optimizing this? file sys-call on every set cmd? file.write_all(serialized.as_bytes())?; - + Ok(()) } /// Get : When retrieving a value for a key with the get command, it searches the index, /// and if found then loads from the log the command at the corresponding log pointer, /// evaluates the command and returns the result. - pub fn get(&self, key: String) -> Result> { + fn get(&self, key: String) -> Result> { if let Some(&offset) = self.map.get(&key) { debug!("GET offset: {:?}", offset); // File reset seek on self.disk @@ -327,7 +339,7 @@ impl KvStore { /// Remove : When removing a key, similarly, kvs writes the rm command in the log, /// Checking to see first that the key exists /// then removes the key from the in-memory index. - pub fn remove(&mut self, key: String) -> Result<()> { + fn remove(&mut self, key: String) -> Result<()> { // Check using in memory map if self.map.contains_key(&key) { let mut file = self @@ -345,8 +357,47 @@ impl KvStore { self.map.remove(&key); Ok(()) } else { - error!("No such key: {:?}", key); + warn!("No such key: {:?}", key); Err(DbError::KeyNotFound) } } } +/// Sled backend for KVS +pub struct SledKvsEngine { + db: sled::Db, +} + +impl SledKvsEngine { + /// Start a Sled Kvs Engine + pub fn open(path: impl Into) -> Result { + let db = sled::open(path.into())?; + Ok(SledKvsEngine { db }) + } +} +impl KvsEngine for SledKvsEngine { + fn set(&mut self, key: String, value: String) -> Result<()> { + let _result = self.db.insert(key.as_bytes(), value.as_bytes())?; + Ok(()) + } + + fn get(&self, key: String) -> Result> { + if let Some(result) = self.db.get(key.as_bytes())? { + return Ok(Some( + String::from_utf8(result.to_vec()) + .map_err(|utf8_err| DbError::SledUtf8Error(utf8_err))?, + )); + } + Ok(None) + } + + fn remove(&mut self, key: String) -> Result<()> { + match self.db.remove(key.as_bytes()) { + Ok(Some(_)) => Ok(()), + Ok(None) => { + warn!("No such key: {:?}", key); + Err(DbError::KeyNotFound) + } + Err(sled_err) => Err(DbError::SledError(sled_err)), + } + } +} diff --git a/lib/src/utils.rs b/lib/src/utils.rs new file mode 100644 index 0000000..6e3a837 --- /dev/null +++ b/lib/src/utils.rs @@ -0,0 +1,6 @@ +//! Common space for utils used across crates + +/// Non-zero exit code indicates a program error +pub fn exit_program(code: i32) -> ! { + std::process::exit(code) +} \ No newline at end of file diff --git a/lib/tests/kv_store.rs b/lib/tests/kv_store.rs new file mode 100644 index 0000000..7c2c553 --- /dev/null +++ b/lib/tests/kv_store.rs @@ -0,0 +1,129 @@ +#![allow(unused_mut)] + +use kvs::{KvStore, KvsEngine, Result}; +use tempfile::TempDir; +use walkdir::WalkDir; + +// Should get previously stored value +#[test] +fn get_stored_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + store.set("key2".to_owned(), "value2".to_owned())?; + + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); + + // Open from disk again and check persistent data + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); + + Ok(()) +} + +// Should overwrite existent value +#[test] +fn overwrite_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + store.set("key1".to_owned(), "value2".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); + + // Open from disk again and check persistent data + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); + store.set("key1".to_owned(), "value3".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value3".to_owned())); + + Ok(()) +} + +// Should get `None` when getting a non-existent key +#[test] +fn get_non_existent_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + assert_eq!(store.get("key2".to_owned())?, None); + + // Open from disk again and check persistent data + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key2".to_owned())?, None); + + Ok(()) +} + +#[test] +fn remove_non_existent_key() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + assert!(store.remove("key1".to_owned()).is_err()); + Ok(()) +} + +#[test] +fn remove_key() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + store.set("key1".to_owned(), "value1".to_owned())?; + assert!(store.remove("key1".to_owned()).is_ok()); + assert_eq!(store.get("key1".to_owned())?, None); + Ok(()) +} + +// Insert data until total size of the directory decreases. +// Test data correctness after compaction. +#[test] +#[ignore = "takes a while, but passes"] +fn compaction() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + let dir_size = || { + let entries = WalkDir::new(temp_dir.path()).into_iter(); + let len: walkdir::Result = entries + .map(|res| { + res.and_then(|entry| entry.metadata()) + .map(|metadata| metadata.len()) + }) + .sum(); + len.expect("fail to get directory size") + }; + + let mut current_size = dir_size(); + for iter in 0..1000 { + for key_id in 0..1000 { + let key = format!("key{}", key_id); + let value = format!("{}", iter); + store.set(key, value)?; + } + + let new_size = dir_size(); + if new_size > current_size { + current_size = new_size; + continue; + } + // Compaction triggered + + drop(store); + // reopen and check content + let mut store = KvStore::open(temp_dir.path())?; + for key_id in 0..1000 { + let key = format!("key{}", key_id); + assert_eq!(store.get(key)?, Some(format!("{}", iter))); + } + return Ok(()); + } + + panic!("No compaction detected"); +} \ No newline at end of file diff --git a/server.sh b/server.sh new file mode 100755 index 0000000..3666aab --- /dev/null +++ b/server.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export RUST_LOG=trace +cargo build --workspace; + +cargo watch --watch crates/kvs-server -x 'r -p kvs-server -- --engine kvs' \ No newline at end of file diff --git a/tests/test.rs b/tests/test.rs deleted file mode 100644 index 46f8d02..0000000 --- a/tests/test.rs +++ /dev/null @@ -1,304 +0,0 @@ -use assert_cmd::prelude::*; -use kvs::{KvStore, Result}; -use predicates::ord::eq; -use predicates::str::{contains, is_empty, PredicateStrExt}; -use std::process::Command; -use tempfile::TempDir; -use walkdir::WalkDir; - -// `kvs` with no args should exit with a non-zero code. -#[test] -fn cli_no_args() { - Command::cargo_bin("kvs").unwrap().assert().failure(); -} - -// `kvs -V` should print the version -#[test] -fn cli_version() { - Command::cargo_bin("kvs") - .unwrap() - .args(&["-V"]) - .assert() - .stdout(contains(env!("CARGO_PKG_VERSION"))); -} - -// `kvs get ` should print "Key not found" for a non-existent key and exit with zero. -#[ignore = "Manually checked. We use logs instead of printlns"] -#[test] -fn cli_get_non_existent_key() { - let temp_dir = TempDir::new().unwrap(); - Command::cargo_bin("kvs") - .unwrap() - .args(&["get", "key1"]) - .current_dir(&temp_dir) - .assert() - .success() - .stdout(eq("Key not found").trim()); -} - -// `kvs rm ` should print "Key not found" for an empty database and exit with non-zero code. -#[test] -#[ignore = "Manually checked. We use logs instead of printlns"] -fn cli_rm_non_existent_key() { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - Command::cargo_bin("kvs") - .unwrap() - .args(&["rm", "key1"]) - .current_dir(&temp_dir) - .assert() - .failure() - .stdout(eq("Key not found").trim()); -} - -// `kvs set ` should print nothing and exit with zero. -#[test] -fn cli_set() { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - Command::cargo_bin("kvs") - .unwrap() - .args(&["set", "key1", "value1"]) - .current_dir(&temp_dir) - .assert() - .success() - .stdout(is_empty()); -} - -#[test] -fn cli_get_stored() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - - let mut store = KvStore::open(temp_dir.path())?; - store.set("key1".to_owned(), "value1".to_owned())?; - store.set("key2".to_owned(), "value2".to_owned())?; - drop(store); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["get", "key1"]) - .current_dir(&temp_dir) - .assert() - .success() - .stdout(eq("value1").trim()); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["get", "key2"]) - .current_dir(&temp_dir) - .assert() - .success() - .stdout(eq("value2").trim()); - - Ok(()) -} - -// `kvs rm ` should print nothing and exit with zero. -#[test] -#[ignore = "Manually checked. We use logs instead of printlns"] -fn cli_rm_stored() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - - let mut store = KvStore::open(temp_dir.path())?; - store.set("key1".to_owned(), "value1".to_owned())?; - drop(store); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["rm", "key1"]) - .current_dir(&temp_dir) - .assert() - .success() - .stdout(is_empty()); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["get", "key1"]) - .current_dir(&temp_dir) - .assert() - .success() - .stdout(eq("Key not found").trim()); - - Ok(()) -} - -#[test] -fn cli_invalid_get() { - Command::cargo_bin("kvs") - .unwrap() - .args(&["get"]) - .assert() - .failure(); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["get", "extra", "field"]) - .assert() - .failure(); -} - -#[test] -fn cli_invalid_set() { - Command::cargo_bin("kvs") - .unwrap() - .args(&["set"]) - .assert() - .failure(); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["set", "missing_field"]) - .assert() - .failure(); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["set", "extra", "extra", "field"]) - .assert() - .failure(); -} - -#[test] -fn cli_invalid_rm() { - Command::cargo_bin("kvs") - .unwrap() - .args(&["rm"]) - .assert() - .failure(); - - Command::cargo_bin("kvs") - .unwrap() - .args(&["rm", "extra", "field"]) - .assert() - .failure(); -} - -#[test] -fn cli_invalid_subcommand() { - Command::cargo_bin("kvs") - .unwrap() - .args(&["unknown", "subcommand"]) - .assert() - .failure(); -} - -// Should get previously stored value. -#[test] -fn get_stored_value() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut store = KvStore::open(temp_dir.path())?; - - store.set("key1".to_owned(), "value1".to_owned())?; - store.set("key2".to_owned(), "value2".to_owned())?; - - assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); - assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); - - // Open from disk again and check persistent data. - drop(store); - let store = KvStore::open(temp_dir.path())?; - assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); - assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); - - Ok(()) -} - -// Should overwrite existent value. -#[test] -fn overwrite_value() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut store = KvStore::open(temp_dir.path())?; - - store.set("key1".to_owned(), "value1".to_owned())?; - assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); - store.set("key1".to_owned(), "value2".to_owned())?; - assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); - - // Open from disk again and check persistent data. - drop(store); - let mut store = KvStore::open(temp_dir.path())?; - assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); - store.set("key1".to_owned(), "value3".to_owned())?; - assert_eq!(store.get("key1".to_owned())?, Some("value3".to_owned())); - - Ok(()) -} - -// Should get `None` when getting a non-existent key. -#[test] -fn get_non_existent_value() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut store = KvStore::open(temp_dir.path())?; - - store.set("key1".to_owned(), "value1".to_owned())?; - assert_eq!(store.get("key2".to_owned())?, None); - - // Open from disk again and check persistent data. - drop(store); - let store = KvStore::open(temp_dir.path())?; - assert_eq!(store.get("key2".to_owned())?, None); - - Ok(()) -} - -#[test] -fn remove_non_existent_key() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut store = KvStore::open(temp_dir.path())?; - assert!(store.remove("key1".to_owned()).is_err()); - Ok(()) -} - -#[test] -fn remove_key() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut store = KvStore::open(temp_dir.path())?; - store.set("key1".to_owned(), "value1".to_owned())?; - assert!(store.remove("key1".to_owned()).is_ok()); - assert_eq!(store.get("key1".to_owned())?, None); - Ok(()) -} - -// Insert data until total size of the directory decreases. -// Test data correctness after compaction. -#[test] -fn compaction() -> Result<()> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut store = KvStore::open(temp_dir.path())?; - - let dir_size = || { - let entries = WalkDir::new(temp_dir.path()).into_iter(); - let len: walkdir::Result = entries - .map(|res| { - res.and_then(|entry| entry.metadata()) - .map(|metadata| metadata.len()) - }) - .sum(); - len.expect("fail to get directory size") - }; - - let mut current_size = dir_size(); - for iter in 0..1000 { - for key_id in 0..1000 { - let key = format!("key{}", key_id); - let value = format!("{}", iter); - store.set(key, value)?; - } - - let new_size = dir_size(); - if new_size > current_size { - current_size = new_size; - continue; - } - // Compaction triggered. - - drop(store); - // reopen and check content. - let store = KvStore::open(temp_dir.path())?; - for key_id in 0..1000 { - let key = format!("key{}", key_id); - assert_eq!(store.get(key)?, Some(format!("{}", iter))); - } - return Ok(()); - } - - panic!("No compaction detected"); -} \ No newline at end of file