Macro for testing all layouts

trie-db/fuzz/Cargo.toml

@@ -28,11 +28,6 @@ members = ["."]
 name = "trie_root_new"
 path = "fuzz_targets/trie_root_new.rs"
 
-[[bin]]
-name = "trie_unhashed_no_ext"
-path = "fuzz_targets/trie_unhashed_no_ext.rs"
-
-
 [[bin]]
 name = "trie_root"
 path = "fuzz_targets/trie_root.rs"
@@ -61,6 +56,10 @@ path = "fuzz_targets/seek_iter.rs"
 name = "trie_proof_valid"
 path = "fuzz_targets/trie_proof_valid.rs"
 
+[[bin]]
+name = "trie_codec_proof"
+path = "fuzz_targets/trie_codec_proof.rs"
+
 [[bin]]
 name = "trie_proof_invalid"
 path = "fuzz_targets/trie_proof_invalid.rs"

trie-db/fuzz/fuzz_targets/no_ext_insert.rs

@@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
 	// fuzzed code goes here
-	fuzz_that_no_extension_insert(data);
+	fuzz_that_no_extension_insert::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs

@@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
 	// fuzzed code goes here
-	fuzz_that_no_extension_insert_remove(data);
+	fuzz_that_no_extension_insert_remove::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/fuzz_targets/prefix_iter.rs

@@ -4,5 +4,5 @@ use trie_db_fuzz::fuzz_prefix_iter;
 use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
-	fuzz_prefix_iter(data);
+	fuzz_prefix_iter::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/fuzz_targets/seek_iter.rs

@@ -4,5 +4,5 @@ use trie_db_fuzz::fuzz_seek_iter;
 use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
-	fuzz_seek_iter(data);
+	fuzz_seek_iter::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/fuzz_targets/trie_codec_proof.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use trie_db_fuzz::fuzz_that_trie_codec_proofs;
+
+fuzz_target!(|data: &[u8]| {
+	fuzz_that_trie_codec_proofs::<reference_trie::ExtensionLayout>(data);
+});

trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs

@@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target;
 use trie_db_fuzz::fuzz_that_verify_rejects_invalid_proofs;
 
 fuzz_target!(|data: &[u8]| {
-	fuzz_that_verify_rejects_invalid_proofs(data);
-});
+	fuzz_that_verify_rejects_invalid_proofs::<reference_trie::ExtensionLayout>(data);
+});

trie-db/fuzz/fuzz_targets/trie_proof_valid.rs

@@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target;
 use trie_db_fuzz::fuzz_that_verify_accepts_valid_proofs;
 
 fuzz_target!(|data: &[u8]| {
-	fuzz_that_verify_accepts_valid_proofs(data);
-});
+	fuzz_that_verify_accepts_valid_proofs::<reference_trie::ExtensionLayout>(data);
+});

trie-db/fuzz/fuzz_targets/trie_root.rs

@@ -5,5 +5,5 @@ use trie_db_fuzz::fuzz_that_reference_trie_root;
 use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
-	fuzz_that_reference_trie_root(data);
+	fuzz_that_reference_trie_root::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs

@@ -5,5 +5,5 @@ use trie_db_fuzz::fuzz_that_reference_trie_root_fix_length;
 use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
-	fuzz_that_reference_trie_root_fix_length(data);
+	fuzz_that_reference_trie_root_fix_length::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/fuzz_targets/trie_root_new.rs

@@ -6,5 +6,5 @@ use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| {
 	// fuzzed code goes here
-	fuzz_that_compare_implementations(data);
+	fuzz_that_compare_implementations::<reference_trie::NoExtensionLayout>(data);
 });

trie-db/fuzz/src/lib.rs

@@ -14,22 +14,17 @@
 
 
 use hash_db::Hasher;
-use keccak_hasher::KeccakHasher;
 use memory_db::{HashKey, MemoryDB, PrefixedKey};
 use reference_trie::{
-	calc_root_no_extension,
-	compare_no_extension_insert_remove,
-	ExtensionLayout,
-	NoExtensionLayout,
+	calc_root,
 	proof::{generate_proof, verify_proof},
-	reference_trie_root,
-	RefTrieDBMut,
-	RefTrieDBMutNoExt,
-	RefTrieDBNoExt,
+	reference_trie_root_iter_build as reference_trie_root,
+	TrieDBMut,
 	TrieDBIterator,
+	compare_insert_remove,
 };
 use std::convert::TryInto;
-use trie_db::{DBValue, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut};
+use trie_db::{DBValue, Trie, TrieDB, TrieLayout, TrieMut};
 
 fn fuzz_to_data(input: &[u8]) -> Vec<(Vec<u8>,Vec<u8>)> {
 	let mut result = Vec::new();
@@ -94,26 +89,26 @@ fn fuzz_removal(data: Vec<(Vec<u8>,Vec<u8>)>) -> Vec<(bool, Vec<u8>,Vec<u8>)> {
 	res
 }
 
-pub fn fuzz_that_reference_trie_root(input: &[u8]) {
+pub fn fuzz_that_reference_trie_root<T: TrieLayout>(input: &[u8]) {
 	let data = data_sorted_unique(fuzz_to_data(input));
 	let mut memdb = MemoryDB::<_, HashKey<_>, _>::default();
 	let mut root = Default::default();
-	let mut t = RefTrieDBMut::new(&mut memdb, &mut root);
+	let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root);
 	for a in 0..data.len() {
 		t.insert(&data[a].0[..], &data[a].1[..]).unwrap();
 	}
-	assert_eq!(*t.root(), reference_trie_root(data));
+	assert_eq!(*t.root(), reference_trie_root::<T, _, _, _>(data));
 }
 
-pub fn fuzz_that_reference_trie_root_fix_length(input: &[u8]) {
+pub fn fuzz_that_reference_trie_root_fix_length<T: TrieLayout>(input: &[u8]) {
 	let data = data_sorted_unique(fuzz_to_data_fix_length(input));
 	let mut memdb = MemoryDB::<_, HashKey<_>, _>::default();
 	let mut root = Default::default();
-	let mut t = RefTrieDBMut::new(&mut memdb, &mut root);
+	let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root);
 	for a in 0..data.len() {
 		t.insert(&data[a].0[..], &data[a].1[..]).unwrap();
 	}
-	assert_eq!(*t.root(), reference_trie_root(data));
+	assert_eq!(*t.root(), reference_trie_root::<T, _, _, _>(data));
 }
 
 fn fuzz_to_data_fix_length(input: &[u8]) -> Vec<(Vec<u8>,Vec<u8>)> {
@@ -141,50 +136,45 @@ fn data_sorted_unique(input: Vec<(Vec<u8>,Vec<u8>)>) -> Vec<(Vec<u8>,Vec<u8>)> {
 	m.into_iter().collect()
 }
 
-pub fn fuzz_that_compare_implementations(input: &[u8]) {
+pub fn fuzz_that_compare_implementations<T: TrieLayout>(input: &[u8]) {
 	let data = data_sorted_unique(fuzz_to_data(input));
 	//println!("data:{:?}", &data);
 	let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default();
-	let hashdb = MemoryDB::<KeccakHasher, PrefixedKey<_>, DBValue>::default();
-	reference_trie::compare_implementations(data, memdb, hashdb);
+	let hashdb = MemoryDB::<T::Hash, PrefixedKey<_>, DBValue>::default();
+	reference_trie::compare_implementations::<T, _>(data, memdb, hashdb);
 }
 
-pub fn fuzz_that_unhashed_no_extension(input: &[u8]) {
-	let data = data_sorted_unique(fuzz_to_data(input));
-	reference_trie::compare_unhashed_no_extension(data);
-}
-
-pub fn fuzz_that_no_extension_insert(input: &[u8]) {
+pub fn fuzz_that_no_extension_insert<T: TrieLayout>(input: &[u8]) {
 	let data = fuzz_to_data(input);
 	//println!("data{:?}", data);
 	let mut memdb = MemoryDB::<_, HashKey<_>, _>::default();
 	let mut root = Default::default();
-	let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root);
+	let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root);
 	for a in 0..data.len() {
 		t.insert(&data[a].0[..], &data[a].1[..]).unwrap();
 	}
 	// we are testing the RefTrie code here so we do not sort or check uniqueness
 	// before.
 	let data = data_sorted_unique(fuzz_to_data(input));
 	//println!("data{:?}", data);
-	assert_eq!(*t.root(), calc_root_no_extension(data));
+	assert_eq!(*t.root(), calc_root::<T, _, _, _>(data));
 }
 
-pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) {
+pub fn fuzz_that_no_extension_insert_remove<T: TrieLayout>(input: &[u8]) {
 	let data = fuzz_to_data(input);
 	let data = fuzz_removal(data);
 
 	let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default();
-	compare_no_extension_insert_remove(data, memdb);
+	compare_insert_remove::<T, _>(data, memdb);
 }
 
-pub fn fuzz_seek_iter(input: &[u8]) {
+pub fn fuzz_seek_iter<T: TrieLayout>(input: &[u8]) {
 	let data = data_sorted_unique(fuzz_to_data_fix_length(input));
 
 	let mut memdb = MemoryDB::<_, HashKey<_>, _>::default();
 	let mut root = Default::default();
 	{
-		let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root);
+		let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root);
 		for a in 0..data.len() {
 			t.insert(&data[a].0[..], &data[a].1[..]).unwrap();
 		}
@@ -203,7 +193,7 @@ pub fn fuzz_seek_iter(input: &[u8]) {
 	let mut iter_res = Vec::new();
 	let mut error = 0;
 	{
-			let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap();
+			let trie = TrieDB::<T>::new(&memdb, &root).unwrap();
 			let mut iter =  trie.iter().unwrap();
 			if let Ok(_) = iter.seek(prefix) {
 			} else {
@@ -227,13 +217,13 @@ pub fn fuzz_seek_iter(input: &[u8]) {
 	assert_eq!(error, 0);
 }
 
-pub fn fuzz_prefix_iter(input: &[u8]) {
+pub fn fuzz_prefix_iter<T: TrieLayout>(input: &[u8]) {
 	let data = data_sorted_unique(fuzz_to_data_fix_length(input));
 
 	let mut memdb = MemoryDB::<_, HashKey<_>, _>::default();
 	let mut root = Default::default();
 	{
-		let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root);
+		let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root);
 		for a in 0..data.len() {
 			t.insert(&data[a].0[..], &data[a].1[..]).unwrap();
 		}
@@ -252,7 +242,7 @@ pub fn fuzz_prefix_iter(input: &[u8]) {
 	let mut iter_res = Vec::new();
 	let mut error = 0;
 	{
-			let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap();
+			let trie = TrieDB::<T>::new(&memdb, &root).unwrap();
 			let iter = TrieDBIterator::new_prefixed(&trie, prefix).unwrap();
 
 			for x in iter {
@@ -273,7 +263,7 @@ pub fn fuzz_prefix_iter(input: &[u8]) {
 	assert_eq!(error, 0);
 }
 
-pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) {
+pub fn fuzz_that_verify_accepts_valid_proofs<T: TrieLayout>(input: &[u8]) {
 	let mut data = fuzz_to_data(input);
 	// Split data into 3 parts:
 	// - the first 1/3 is added to the trie and not included in the proof
@@ -289,11 +279,30 @@ pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) {
 	keys.sort();
 	keys.dedup();
 
-	let (root, proof, items) = test_generate_proof::<ExtensionLayout>(data, keys);
-	assert!(verify_proof::<ExtensionLayout, _, _, _>(&root, &proof, items.iter()).is_ok());
+	let (root, proof, items) = test_generate_proof::<T>(data, keys);
+	assert!(verify_proof::<T, _, _, _>(&root, &proof, items.iter()).is_ok());
 }
 
-pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) {
+pub fn fuzz_that_trie_codec_proofs<T: TrieLayout>(input: &[u8]) {
+	let mut data = fuzz_to_data(input);
+	// Split data into 3 parts:
+	// - the first 1/3 is added to the trie and not included in the proof
+	// - the second 1/3 is added to the trie and included in the proof
+	// - the last 1/3 is not added to the trie and the proof proves non-inclusion of them
+	let mut keys = data[(data.len() / 3)..]
+		.iter()
+		.map(|(key, _)| key.clone())
+		.collect::<Vec<_>>();
+	data.truncate(data.len() * 2 / 3);
+
+	let data = data_sorted_unique(data);
+	keys.sort();
+	keys.dedup();
+
+	test_trie_codec_proof::<T>(data, keys);
+}
+
+pub fn fuzz_that_verify_rejects_invalid_proofs<T: TrieLayout>(input: &[u8]) {
 	if input.len() < 4 {
 		return;
 	}
@@ -321,7 +330,7 @@ pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) {
 		return;
 	}
 
-	let (root, proof, mut items) = test_generate_proof::<ExtensionLayout>(data, keys);
+	let (root, proof, mut items) = test_generate_proof::<T>(data, keys);
 
 	// Make one item at random incorrect.
 	let items_idx = random_int % items.len();
@@ -330,7 +339,7 @@ pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) {
 		(_, value) if value.is_some() => *value = None,
 		(_, value) => *value = Some(DBValue::new()),
 	}
-	assert!(verify_proof::<ExtensionLayout, _, _, _>(&root, &proof, items.iter()).is_err());
+	assert!(verify_proof::<T, _, _, _>(&root, &proof, items.iter()).is_err());
 }
 
 fn test_generate_proof<L: TrieLayout>(
@@ -363,3 +372,65 @@ fn test_generate_proof<L: TrieLayout>(
 
 	(root, proof, items)
 }
+
+fn test_trie_codec_proof<L: TrieLayout>(
+	entries: Vec<(Vec<u8>, Vec<u8>)>,
+	keys: Vec<Vec<u8>>,
+)
+{
+	use hash_db::{HashDB, EMPTY_PREFIX};
+	use reference_trie::{
+		Recorder,
+		encode_compact, decode_compact,
+	};
+
+	// Populate DB with full trie from entries.
+	let (db, root) = {
+		let mut db = <MemoryDB<L::Hash, HashKey<_>, _>>::default();
+		let mut root = Default::default();
+		{
+			let mut trie = <TrieDBMut<L>>::new(&mut db, &mut root);
+			for (key, value) in entries {
+				trie.insert(&key, &value).unwrap();
+			}
+		}
+		(db, root)
+	};
+	let expected_root = root;
+	// Lookup items in trie while recording traversed nodes.
+	let mut recorder = Recorder::new();
+	let items = {
+		let mut items = Vec::with_capacity(keys.len());
+		let trie = <TrieDB<L>>::new(&db, &root).unwrap();
+		for key in keys {
+			let value = trie.get_with(key.as_slice(), &mut recorder).unwrap();
+			items.push((key, value));
+		}
+		items
+	};
+
+	// Populate a partial trie DB with recorded nodes.
+	let mut partial_db = <MemoryDB<L::Hash, HashKey<_>, _>>::default();
+	for record in recorder.drain() {
+		partial_db.emplace(record.hash, EMPTY_PREFIX, record.data);
+	}
+
+	// Compactly encode the partial trie DB.
+	let compact_trie = {
+		let trie = <TrieDB<L>>::new(&partial_db, &root).unwrap();
+		encode_compact::<L>(&trie).unwrap()
+	};
+
+	let expected_used = compact_trie.len();
+	// Reconstruct the partial DB from the compact encoding.
+	let mut db = <MemoryDB<L::Hash, HashKey<_>, _>>::default();
+	let (root, used) = decode_compact::<L, _, _>(&mut db, &compact_trie).unwrap();
+	assert_eq!(root, expected_root);
+	assert_eq!(used, expected_used);
+
+	// Check that lookups for all items succeed.
+	let trie = <TrieDB<L>>::new(&db, &root).unwrap();
+	for (key, expected_value) in items {
+		assert_eq!(trie.get(key.as_slice()).unwrap(), expected_value);
+	}
+}