From 81d0192acf772df1dbff37993a5f77e170ae723d Mon Sep 17 00:00:00 2001 From: okilisan Date: Fri, 24 May 2024 17:42:35 +0200 Subject: [PATCH] fix the fishhash & fishhashplus implementations --- Cargo.lock | 1 + consensus/pow/src/lib.rs | 29 +- .../pre_ghostdag_validation.rs | 1 + crypto/hashes/Cargo.toml | 1 + crypto/hashes/src/pow_hashers.rs | 291 +++++++++++++----- 5 files changed, 237 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1b7956..f0d8923 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2908,6 +2908,7 @@ dependencies = [ "js-sys", "karlsen-utils", "keccak", + "lazy_static", "once_cell", "rand 0.8.5", "serde", diff --git a/consensus/pow/src/lib.rs b/consensus/pow/src/lib.rs index 5d38701..9537a8e 100644 --- a/consensus/pow/src/lib.rs +++ b/consensus/pow/src/lib.rs @@ -21,7 +21,7 @@ pub struct State { pub(crate) target: Uint256, // PRE_POW_HASH || TIME || 32 zero byte padding; without NONCE pub(crate) hasher: PowB3Hash, - pub(crate) fishhasher: PowFishHash, + //pub(crate) fishhasher: PowFishHash, pub(crate) header_version: u16, } @@ -35,10 +35,10 @@ impl State { //let hasher = PowHash::new(pre_pow_hash, header.timestamp); let hasher = PowB3Hash::new(pre_pow_hash, header.timestamp); let matrix = Matrix::generate(pre_pow_hash); - let fishhasher = PowFishHash::new(); + //let fishhasher = PowFishHash::new(); let header_version = header.version; - Self { matrix, target, hasher, fishhasher, header_version} + Self { matrix, target, hasher, /*fishhasher,*/ header_version} } @@ -52,7 +52,24 @@ impl State { fn calculate_pow_khashv2(&self, nonce: u64) -> Uint256 { // Hasher already contains PRE_POW_HASH || TIME || 32 zero byte padding; so only the NONCE is missing let hash = self.hasher.clone().finalize_with_nonce(nonce); - let hash = self.matrix.heavy_hash(hash); + //println!("hash-1 : {:?}", hash); + let hash = PowFishHash::fishhash_kernel(&hash); + //println!("hash-2 : {:?}", hash); + //last b3 hash + let hash = PowB3Hash::hash(hash); + //println!("hash-3 : {:?}", hash); + Uint256::from_le_bytes(hash.as_bytes()) + } + + fn calculate_pow_khashv2plus(&self, nonce: u64) -> Uint256 { + // Hasher already contains PRE_POW_HASH || TIME || 32 zero byte padding; so only the NONCE is missing + let hash = self.hasher.clone().finalize_with_nonce(nonce); + //println!("hash-1 : {:?}", hash); + let hash = PowFishHash::fishhashplus_kernel(&hash); + //println!("hash-2 : {:?}", hash); + //last b3 hash + let hash = PowB3Hash::hash(hash); + //println!("hash-3 : {:?}", hash); Uint256::from_le_bytes(hash.as_bytes()) } @@ -61,9 +78,9 @@ impl State { /// PRE_POW_HASH || TIME || 32 zero byte padding || NONCE pub fn calculate_pow(&self, nonce: u64) -> Uint256 { if self.header_version == constants::BLOCK_VERSION { - return self.calculate_pow_khashv1(nonce); - } else if self.header_version == constants::BLOCK_VERSION_KHASHV2 { return self.calculate_pow_khashv2(nonce); + } else if self.header_version == constants::BLOCK_VERSION_KHASHV2 { + return self.calculate_pow_khashv2plus(nonce); } else { // TODO handle block version error //Err(RuleError::WrongBlockVersion(self.header_version)); diff --git a/consensus/src/pipeline/header_processor/pre_ghostdag_validation.rs b/consensus/src/pipeline/header_processor/pre_ghostdag_validation.rs index 2d9a1d0..82554f1 100644 --- a/consensus/src/pipeline/header_processor/pre_ghostdag_validation.rs +++ b/consensus/src/pipeline/header_processor/pre_ghostdag_validation.rs @@ -34,6 +34,7 @@ impl HeaderProcessor { Ok(()) } + // TODO : setup dual block version managment fn check_header_version(&self, header: &Header) -> BlockProcessResult<()> { if header.version != constants::BLOCK_VERSION_KHASHV2 { return Err(RuleError::WrongBlockVersion(header.version)); diff --git a/crypto/hashes/Cargo.toml b/crypto/hashes/Cargo.toml index 84bf458..6ef58de 100644 --- a/crypto/hashes/Cargo.toml +++ b/crypto/hashes/Cargo.toml @@ -13,6 +13,7 @@ repository.workspace = true no-asm = ["keccak"] [dependencies] +lazy_static = "1.4.0" blake2b_simd.workspace = true borsh.workspace = true faster-hex.workspace = true diff --git a/crypto/hashes/src/pow_hashers.rs b/crypto/hashes/src/pow_hashers.rs index d5c59bc..045141a 100644 --- a/crypto/hashes/src/pow_hashers.rs +++ b/crypto/hashes/src/pow_hashers.rs @@ -2,6 +2,8 @@ use crate::Hash; use std::ops::BitXor; use tiny_keccak::Hasher; +use lazy_static::lazy_static; + #[derive(Clone)] pub struct PowB3Hash{ pub hasher: blake3::Hasher, @@ -16,12 +18,12 @@ pub struct KHeavyHash; #[derive(Clone)] pub struct PowFishHash{ // set the cache here not hasher - pub context: Context, + //pub context: Context, } const FNV_PRIME: u32 = 0x01000193; const FULL_DATASET_ITEM_PARENTS: u32 = 512; -const NUM_DATASET_ACCESSES: i32 = 32; +const NUM_DATASET_ACCESSES: u32 = 32; const LIGHT_CACHE_ROUNDS: i32 = 3; const LIGHT_CACHE_NUM_ITEMS: u32 = 1179641; @@ -37,6 +39,7 @@ const SIZE_U64: usize = std::mem::size_of::(); pub trait HashData { fn new() -> Self; + fn from_hash(hash: &Hash) -> Self; fn as_bytes(&self) -> &[u8]; fn as_bytes_mut(&mut self) -> &mut [u8]; @@ -75,6 +78,10 @@ impl HashData for Hash256 { Self([0; 32]) } + fn from_hash(hash: &Hash) -> Self { + Self(hash.0) + } + fn as_bytes(&self) -> &[u8] { &self.0 } @@ -92,6 +99,14 @@ impl HashData for Hash512 { Self([0; 64]) } + //Todo check if filled with 0 + fn from_hash(hash: &Hash) -> Self { + let mut result = Self::new(); + let (first_half, second_half) = result.0.split_at_mut(hash.0.len()); + first_half.copy_from_slice(&hash.0); + result + } + fn as_bytes(&self) -> &[u8] { &self.0 } @@ -123,6 +138,14 @@ impl HashData for Hash1024 { Self([0; 128]) } + //Todo check if filled with 0 + fn from_hash(hash: &Hash) -> Self { + let mut result = Self::new(); + let (first_half, second_half) = result.0.split_at_mut(hash.0.len()); + first_half.copy_from_slice(&hash.0); + result + } + fn as_bytes(&self) -> &[u8] { &self.0 } @@ -141,12 +164,39 @@ impl Hash1024 { hash } + + fn from_256s(first: &Hash256, second: &Hash256) -> Self { + //filled with 0 by default + let mut hash = Self::new(); + let (first_half, second_half) = hash.0.split_at_mut(first.0.len() * 2); + first_half.copy_from_slice(&first.0); + second_half.copy_from_slice(&second.0); + + hash + } } #[derive(Clone)] pub struct Context { - pub light_cache: Box<[Hash512]>, - pub full_dataset: Option>, + //pub light_cache: Box<[Hash512]>, + //pub full_dataset: Option>, +} + +lazy_static! { + static ref LIGHT_CACHE: Box<[Hash512]> = { + //vec![Hash512::new(); LIGHT_CACHE_NUM_ITEMS as usize].into_boxed_slice() + println!("light cache processing started"); + let mut light_cache = + vec![Hash512::new(); LIGHT_CACHE_NUM_ITEMS as usize].into_boxed_slice(); + println!("light_cache[10] : {:?}", light_cache[10]); + println!("light_cache[42] : {:?}", light_cache[42]); + Context::build_light_cache(&mut light_cache); + println!("light_cache[10] : {:?}", light_cache[10]); + println!("light_cache[42] : {:?}", light_cache[42]); + println!("light cache processing done"); + light_cache + }; + static ref INITIALIZED: bool = false; } impl Context { @@ -154,19 +204,10 @@ impl Context { // Vec into boxed sliced, because you can't allocate an array directly on // the heap in rust // https://stackoverflow.com/questions/25805174/creating-a-fixed-size-array-on-heap-in-rust/68122278#68122278 - let mut light_cache = - vec![Hash512::new(); LIGHT_CACHE_NUM_ITEMS as usize].into_boxed_slice(); - Context::build_light_cache(&mut light_cache); - - let full_dataset = if full { - Some(vec![Hash1024::new(); FULL_DATASET_NUM_ITEMS as usize].into_boxed_slice()) - } else { - None - }; Context { - light_cache, - full_dataset, + //light_cache : *LIGHT_CACHE , + //full_dataset, } } @@ -205,9 +246,144 @@ impl PowFishHash { #[inline] - pub fn new() -> Self { - let mut context = Context::new(false); - Self { context } + //pub fn fishhash_kernel(context: &mut Context, seed: &Hash512) -> Hash256 { + pub fn fishhash_kernel(seed: &Hash) -> Hash { + let seed_hash512 = Hash512::from_hash(seed); + let mut mix = Hash1024::from_512s(&seed_hash512, &seed_hash512); + // Fishhash + + for _ in 0..NUM_DATASET_ACCESSES as usize { + // Calculate new fetching indexes + let p0 = mix.get_as_u32(0) % FULL_DATASET_NUM_ITEMS; + let p1 = mix.get_as_u32(4) % FULL_DATASET_NUM_ITEMS; + let p2 = mix.get_as_u32(8) % FULL_DATASET_NUM_ITEMS; + /* + // FishhashPlus + for i in 0..NUM_DATASET_ACCESSES { + // Calculate new fetching indexes + let mut mix_group: [u32; 8] = [0; 8]; + + for (c, mix_group_elem) in mix_group.iter_mut().enumerate() { + *mix_group_elem = mix.get_as_u32(4 * c) + ^ mix.get_as_u32(4 * c + 1) + ^ mix.get_as_u32(4 * c + 2) + ^ mix.get_as_u32(4 * c + 3); + } + + let p0 = (mix_group[0] ^ mix_group[3] ^ mix_group[6]) % FULL_DATASET_NUM_ITEMS; + let p1 = (mix_group[1] ^ mix_group[4] ^ mix_group[7]) % FULL_DATASET_NUM_ITEMS; + let p2 = (mix_group[2] ^ mix_group[5] ^ i) % FULL_DATASET_NUM_ITEMS; + */ + /* + let fetch0 = PowFishHash::lookup(context, p0 as usize); + let mut fetch1 = PowFishHash::lookup(context, p1 as usize); + let mut fetch2 = PowFishHash::lookup(context, p2 as usize); + */ + let fetch0 = PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE,p0 as usize); + let mut fetch1 = PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE,p1 as usize); + let mut fetch2 = PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE,p2 as usize); + + // Modify fetch1 and fetch2 + for j in 0..32 { + fetch1.set_as_u32(j, PowFishHash::fnv1(mix.get_as_u32(j), fetch1.get_as_u32(j))); + fetch2.set_as_u32(j, mix.get_as_u32(j) ^ fetch2.get_as_u32(j)); + } + + // Final computation of new mix + for j in 0..16 { + mix.set_as_u64( + j, + //fetch0.get_as_u64(j) * fetch1.get_as_u64(j) + fetch2.get_as_u64(j), + fetch0 + .get_as_u64(j) + .wrapping_mul(fetch1.get_as_u64(j)) + .wrapping_add(fetch2.get_as_u64(j)), + ); + } + } + + // Collapse the result into 32 bytes + let mut mix_hash = Hash256::new(); + let num_words = std::mem::size_of_val(&mix) / SIZE_U32; + + for i in (0..num_words).step_by(4) { + let h1 = PowFishHash::fnv1(mix.get_as_u32(i), mix.get_as_u32(i + 1)); + let h2 = PowFishHash::fnv1(h1, mix.get_as_u32(i + 2)); + let h3 = PowFishHash::fnv1(h2, mix.get_as_u32(i + 3)); + mix_hash.set_as_u32(i / 4, h3); + } + + Hash::from_bytes(mix_hash.0) + } + + #[inline] + //pub fn fishhash_kernel(context: &mut Context, seed: &Hash512) -> Hash256 { + pub fn fishhashplus_kernel(seed: &Hash) -> Hash { + let seed_hash512 = Hash512::from_hash(seed); + let mut mix = Hash1024::from_512s(&seed_hash512, &seed_hash512); + // Fishhash + /* + for _ in 0..NUM_DATASET_ACCESSES as usize { + // Calculate new fetching indexes + let p0 = mix.get_as_u32(0) % FULL_DATASET_NUM_ITEMS; + let p1 = mix.get_as_u32(4) % FULL_DATASET_NUM_ITEMS; + let p2 = mix.get_as_u32(8) % FULL_DATASET_NUM_ITEMS; + */ + // FishhashPlus + for i in 0..NUM_DATASET_ACCESSES { + // Calculate new fetching indexes + let mut mix_group: [u32; 8] = [0; 8]; + + for (c, mix_group_elem) in mix_group.iter_mut().enumerate() { + *mix_group_elem = mix.get_as_u32(4 * c) + ^ mix.get_as_u32(4 * c + 1) + ^ mix.get_as_u32(4 * c + 2) + ^ mix.get_as_u32(4 * c + 3); + } + + let p0 = (mix_group[0] ^ mix_group[3] ^ mix_group[6]) % FULL_DATASET_NUM_ITEMS; + let p1 = (mix_group[1] ^ mix_group[4] ^ mix_group[7]) % FULL_DATASET_NUM_ITEMS; + let p2 = (mix_group[2] ^ mix_group[5] ^ i) % FULL_DATASET_NUM_ITEMS; + /* + let fetch0 = PowFishHash::lookup(context, p0 as usize); + let mut fetch1 = PowFishHash::lookup(context, p1 as usize); + let mut fetch2 = PowFishHash::lookup(context, p2 as usize); + */ + let fetch0 = PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE,p0 as usize); + let mut fetch1 = PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE,p1 as usize); + let mut fetch2 = PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE,p2 as usize); + + // Modify fetch1 and fetch2 + for j in 0..32 { + fetch1.set_as_u32(j, PowFishHash::fnv1(mix.get_as_u32(j), fetch1.get_as_u32(j))); + fetch2.set_as_u32(j, mix.get_as_u32(j) ^ fetch2.get_as_u32(j)); + } + + // Final computation of new mix + for j in 0..16 { + mix.set_as_u64( + j, + //fetch0.get_as_u64(j) * fetch1.get_as_u64(j) + fetch2.get_as_u64(j), + fetch0 + .get_as_u64(j) + .wrapping_mul(fetch1.get_as_u64(j)) + .wrapping_add(fetch2.get_as_u64(j)), + ); + } + } + + // Collapse the result into 32 bytes + let mut mix_hash = Hash256::new(); + let num_words = std::mem::size_of_val(&mix) / SIZE_U32; + + for i in (0..num_words).step_by(4) { + let h1 = PowFishHash::fnv1(mix.get_as_u32(i), mix.get_as_u32(i + 1)); + let h2 = PowFishHash::fnv1(h1, mix.get_as_u32(i + 2)); + let h3 = PowFishHash::fnv1(h2, mix.get_as_u32(i + 3)); + mix_hash.set_as_u32(i / 4, h3); + } + + Hash::from_bytes(mix_hash.0) } pub fn keccak(out: &mut [u8], data: &[u8]) { @@ -224,7 +400,7 @@ impl PowFishHash { } fn fnv1(u: u32, v: u32) -> u32 { - (u * FNV_PRIME) ^ v + u.wrapping_mul(FNV_PRIME) ^ v } fn fnv1_512(u: Hash512, v: Hash512) -> Hash512 { @@ -267,7 +443,14 @@ impl PowFishHash { Hash1024::from_512s(&mix0, &mix1) } + /*fn lookup(index: usize) -> Hash1024 { + PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE, index) + }*/ + fn lookup(context: &mut Context, index: usize) -> Hash1024 { + //PowFishHash::calculate_dataset_item_1024(&context.light_cache, index); + PowFishHash::calculate_dataset_item_1024(&*LIGHT_CACHE, index) + /* match &mut context.full_dataset { Some(dataset) => { let item = &mut dataset[index]; @@ -279,73 +462,14 @@ impl PowFishHash { } None => PowFishHash::calculate_dataset_item_1024(&context.light_cache, index), } - } - - - fn fishhash_kernel(context: &mut Context, seed: &Hash512) -> Hash256 { - let mut mix = Hash1024::from_512s(seed, seed); - - for _ in 0..NUM_DATASET_ACCESSES as usize { - // Calculate new fetching indexes - let p0 = mix.get_as_u32(0) % FULL_DATASET_NUM_ITEMS; - let p1 = mix.get_as_u32(4) % FULL_DATASET_NUM_ITEMS; - let p2 = mix.get_as_u32(8) % FULL_DATASET_NUM_ITEMS; - - let fetch0 = PowFishHash::lookup(context, p0 as usize); - let mut fetch1 = PowFishHash::lookup(context, p1 as usize); - let mut fetch2 = PowFishHash::lookup(context, p2 as usize); - - // Modify fetch1 and fetch2 - for j in 0..32 { - fetch1.set_as_u32(j, PowFishHash::fnv1(mix.get_as_u32(j), fetch1.get_as_u32(j))); - fetch2.set_as_u32(j, mix.get_as_u32(j) ^ fetch2.get_as_u32(j)); - } - - // Final computation of new mix - for j in 0..16 { - mix.set_as_u64( - j, - fetch0.get_as_u64(j) * fetch1.get_as_u64(j) + fetch2.get_as_u64(j), - ); - } - } - - // Collapse the result into 32 bytes - let mut mix_hash = Hash256::new(); - let num_words = std::mem::size_of_val(&mix) / SIZE_U32; - - for i in (0..num_words).step_by(4) { - let h1 = PowFishHash::fnv1(mix.get_as_u32(i), mix.get_as_u32(i + 1)); - let h2 = PowFishHash::fnv1(h1, mix.get_as_u32(i + 2)); - let h3 = PowFishHash::fnv1(h2, mix.get_as_u32(i + 3)); - mix_hash.set_as_u32(i / 4, h3); - } - - mix_hash + */ } - #[inline] - pub fn hash(output: &mut [u8], context: &mut Context, header: &[u8]) { - let mut seed: Hash512 = Hash512::new(); - - let mut hasher = blake3::Hasher::new(); - hasher.update(header); - let mut output_reader = hasher.finalize_xof(); - output_reader.fill(&mut seed.0); - - let mix_hash = PowFishHash::fishhash_kernel(context, &seed); - - let mut final_data: [u8; 96] = [0; 96]; - final_data[0..64].copy_from_slice(&seed.0); - final_data[64..].copy_from_slice(&mix_hash.0); - - let hash = blake3::hash(&final_data); - output.copy_from_slice(hash.as_bytes()); - } } + impl PowB3Hash { #[inline] @@ -367,6 +491,13 @@ impl PowB3Hash { } + pub fn hash(my_hash: Hash) -> Hash { + let mut hasher = blake3::Hasher::new(); + hasher.update(&my_hash.as_bytes()); + let hash = hasher.finalize(); + Hash(*hash.as_bytes()) + } + } impl PowHash {