From d1c630fe4bd16d1a25d584de9a06bc788e8aa0f3 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 14 Jul 2024 15:39:22 +0100 Subject: [PATCH 01/40] RelativeEncoding, RelativeDecoder, Path <> Path encoding --- data-model/src/encoding/mod.rs | 1 + data-model/src/encoding/relativity.rs | 119 ++++++++++++++++++ data-model/src/path.rs | 16 ++- fuzz/Cargo.toml | 14 +++ fuzz/fuzz_targets/path_rel_path_encoding.rs | 32 +++++ .../path_rel_path_encoding_random.rs | 21 ++++ fuzz/src/lib.rs | 81 +++++++++++- 7 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 data-model/src/encoding/relativity.rs create mode 100644 fuzz/fuzz_targets/path_rel_path_encoding.rs create mode 100644 fuzz/fuzz_targets/path_rel_path_encoding_random.rs diff --git a/data-model/src/encoding/mod.rs b/data-model/src/encoding/mod.rs index 5008409..717bac0 100644 --- a/data-model/src/encoding/mod.rs +++ b/data-model/src/encoding/mod.rs @@ -2,4 +2,5 @@ pub mod compact_width; pub mod error; pub mod max_power; pub mod parameters; +pub mod relativity; pub mod unsigned_int; diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs new file mode 100644 index 0000000..2750e1a --- /dev/null +++ b/data-model/src/encoding/relativity.rs @@ -0,0 +1,119 @@ +use std::future::Future; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; + +use crate::{ + encoding::{ + error::DecodeError, + max_power::{decode_max_power, encode_max_power}, + parameters::{Decoder, Encoder}, + }, + path::{Path, PathRc}, +}; + +/// A relationship representing a type `T` being encoded relative to type `R`. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct RelativeEncoding +where + RelativeEncoding: Encoder, + R: RelativeDecoder, +{ + /// The value we wish to encode or decode. + pub subject: T, + /// The reference value we are encoding the item relative to, usually known by whoever will decode the relative encoding. + pub reference: R, +} + +/// A type that can be used to decode from a bytestring *encoded relative to `Self`*. +pub trait RelativeDecoder { + /// A function from the set of bytestrings *encoded relative to `Self`* to the set of `T` in relation to `Self`. + fn relative_decode( + &self, + producer: &mut Producer, + ) -> impl Future>> + where + Producer: BulkProducer, + Self: Sized; +} + +// Path <> Path + +impl + RelativeEncoding, PathRc> +{ + pub fn new(path: PathRc, reference: PathRc) -> Self { + RelativeEncoding { + subject: path, + reference, + } + } +} + +impl Encoder + for RelativeEncoding, PathRc> +{ + /// Encode a path relative to another path. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_path_relative) + async fn encode( + &self, + consumer: &mut Consumer, + ) -> Result<(), super::error::EncodingConsumerError> + where + Consumer: BulkConsumer, + { + let lcp = self.subject.longest_common_prefix(&self.reference); + encode_max_power(lcp.component_count(), MCC, consumer).await?; + + let suffix = self + .subject + .create_suffix(self.subject.component_count() - lcp.component_count()); + suffix.encode(consumer).await?; + + Ok(()) + } +} + +impl RelativeDecoder> + for PathRc +{ + /// Decode a path relative to this path. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_path_relative) + async fn relative_decode( + &self, + producer: &mut Producer, + ) -> Result> + where + Producer: BulkProducer, + Self: Sized, + { + let lcp = decode_max_power(MCC, producer).await?; + + if lcp > self.component_count() as u64 { + return Err(DecodeError::InvalidInput); + } + + // What is this weird situation? + // LCP is zero, but there IS overlap between + + let prefix = self.create_prefix(lcp as usize); + let suffix = PathRc::::decode(producer).await?; + + let mut new = prefix; + + for component in suffix.components() { + match new.append(component.clone()) { + Ok(appended) => new = appended, + Err(_) => return Err(DecodeError::InvalidInput), + } + } + + let actual_lcp = self.longest_common_prefix(&new); + + if actual_lcp.component_count() != lcp as usize { + return Err(DecodeError::InvalidInput); + } + + Ok(new) + } +} diff --git a/data-model/src/path.rs b/data-model/src/path.rs index 0109633..41b9977 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -151,9 +151,12 @@ pub trait Path: PartialEq + Eq + PartialOrd + Ord + Clone { self.component_count() == 0 } - /// Create a new [`Path`] by taking the first `length` components of this path. + /// Create a new [`Path`] by taking the *first* `length` components of this path. fn create_prefix(&self, length: usize) -> Self; + /// Create a new [`Path`] by taking the *last* `length` components of this path. + fn create_suffix(&self, length: usize) -> Self; + /// Return all possible prefixes of a path, including the empty path and the path itself. fn all_prefixes(&self) -> impl Iterator { let self_len = self.components().count(); @@ -347,6 +350,17 @@ impl Path for PathRc Self { + if length == 0 { + return Self::empty(); + } + + let from = core::cmp::max(0, self.0.len() - length); + let slice = &self.0[from..]; + + Path::new(slice).unwrap() + } + fn append(&self, component: Self::Component) -> Result { let total_component_count = self.0.len(); diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index cf97e6b..c3ff599 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -145,3 +145,17 @@ path = "fuzz_targets/u64be_encoding_random.rs" test = false doc = false bench = false + +[[bin]] +name = "path_rel_path_encoding" +path = "fuzz_targets/path_rel_path_encoding.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "path_rel_path_encoding_random" +path = "fuzz_targets/path_rel_path_encoding_random.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/path_rel_path_encoding.rs b/fuzz/fuzz_targets/path_rel_path_encoding.rs new file mode 100644 index 0000000..a6de5bb --- /dev/null +++ b/fuzz/fuzz_targets/path_rel_path_encoding.rs @@ -0,0 +1,32 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use willow_data_model::{encoding::relativity::RelativeEncoding, path::PathRc}; +use willow_data_model_fuzz::relative_encoding_roundtrip; + +const MCL: usize = 16; +const MCC: usize = 16; +const MPL: usize = 16; + +fuzz_target!(|data: ( + PathRc, + PathRc, + TestConsumer +)| { + let (path_sub, path_ref, mut consumer) = data; + + let relativity = RelativeEncoding { + subject: path_sub, + reference: path_ref, + }; + + smol::block_on(async { + relative_encoding_roundtrip::< + PathRc, + PathRc, + TestConsumer, + >(relativity, &mut consumer) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/path_rel_path_encoding_random.rs b/fuzz/fuzz_targets/path_rel_path_encoding_random.rs new file mode 100644 index 0000000..fd470b4 --- /dev/null +++ b/fuzz/fuzz_targets/path_rel_path_encoding_random.rs @@ -0,0 +1,21 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use willow_data_model::path::PathRc; +use willow_data_model_fuzz::relative_encoding_random; + +const MCL: usize = 16; +const MCC: usize = 16; +const MPL: usize = 16; + +fuzz_target!(|data: (&[u8], PathRc)| { + let (random_bytes, ref_path) = data; + + smol::block_on(async { + relative_encoding_random::, PathRc>( + ref_path, + random_bytes, + ) + .await; + }); +}); diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 6af92c5..626edec 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -10,6 +10,7 @@ use willow_data_model::{ encoding::{ error::DecodeError, parameters::{Decoder, Encoder}, + relativity::{RelativeDecoder, RelativeEncoding}, }, path::*, }; @@ -39,9 +40,9 @@ where let mut producer = FromVec::new(new_vec); // Check for correct errors - let decoded_path = T::decode(&mut producer).await.unwrap(); + let decoded_item = T::decode(&mut producer).await.unwrap(); - assert_eq!(decoded_path, item); + assert_eq!(decoded_item, item); } pub async fn encoding_random(data: &[u8]) @@ -75,6 +76,82 @@ where }; } +pub async fn relative_encoding_roundtrip( + relativity: RelativeEncoding, + consumer: &mut TestConsumer, +) where + T: std::fmt::Debug + PartialEq + Eq, + R: RelativeDecoder, + C: BulkConsumer, + RelativeEncoding: Encoder, +{ + let consumer_should_error = consumer.should_error(); + + if let Err(_err) = relativity.encode(consumer).await { + assert!(consumer_should_error); + return; + } + + if let Err(_err) = consumer.flush().await { + assert!(consumer_should_error); + return; + } + + let mut new_vec = Vec::new(); + + new_vec.extend_from_slice(consumer.as_ref()); + + // THis should eventually be a testproducer, when we are able to initialise one with some known data. + let mut producer = FromVec::new(new_vec); + + // Check for correct errors + let decoded_item = relativity + .reference + .relative_decode(&mut producer) + .await + .unwrap(); + + assert_eq!(decoded_item, relativity.subject); +} + +pub async fn relative_encoding_random(reference: R, data: &[u8]) +where + T: std::fmt::Debug, + R: RelativeDecoder, + RelativeEncoding: Encoder, +{ + let mut producer = SliceProducer::new(data); + + match reference.relative_decode(&mut producer).await { + Ok(item) => { + // It decoded to a valid item! Gasp! + // Can we turn it back into the same encoding? + let mut consumer = IntoVec::::new(); + + let relativity = RelativeEncoding { + subject: item, + reference, + }; + + relativity.encode(&mut consumer).await.unwrap(); + + let encoded = consumer.as_ref().as_slice(); + + assert_eq!(encoded, &data[0..producer.get_offset()]); + } + Err(err) => match err { + // There was an error. + DecodeError::Producer(_) => panic!("Returned producer error, when whe shouldn't!"), + DecodeError::InvalidInput => { + // GOOD. + } + DecodeError::U64DoesNotFitUsize => { + panic!("Returned u64DoesNotFitUsize error, when we shouldn't!") + } + }, + }; +} + pub fn test_successor( baseline: PathRc, candidate: PathRc, From af1083a0d5330a87503db66d9e88157d3e58dfd0 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 14 Jul 2024 16:00:21 +0100 Subject: [PATCH 02/40] Add RelativeEncoder, RelativeEncodeError --- data-model/src/encoding/relativity.rs | 64 +++++++++++---------- fuzz/fuzz_targets/path_rel_path_encoding.rs | 9 +-- fuzz/src/lib.rs | 30 ++++------ 3 files changed, 46 insertions(+), 57 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 2750e1a..bc89606 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -3,24 +3,38 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ - error::DecodeError, + error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, parameters::{Decoder, Encoder}, }, path::{Path, PathRc}, }; -/// A relationship representing a type `T` being encoded relative to type `R`. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct RelativeEncoding -where - RelativeEncoding: Encoder, - R: RelativeDecoder, -{ - /// The value we wish to encode or decode. - pub subject: T, - /// The reference value we are encoding the item relative to, usually known by whoever will decode the relative encoding. - pub reference: R, +/// Returned when a relative encoding fails +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RelativeEncodeError { + /// The subject could not be encoded relative to the reference (e.g. because it was not logically included by the reference). + IllegalRelativity(), + /// The encoding failed to be consumed by a [`ufotofu::local_nb::Consumer`]. + Consumer(EncodingConsumerError), +} + +impl From> for RelativeEncodeError { + fn from(err: EncodingConsumerError) -> Self { + RelativeEncodeError::Consumer(err) + } +} + +/// A type that can be used to encoded to a bytestring *encoded relative to `R`*. +pub trait RelativeEncoder { + /// A function from the set `Self` to the set of bytestrings *encoded relative to `reference`*. + fn relative_encode( + &self, + reference: &R, + consumer: &mut Consumer, + ) -> impl Future>> + where + Consumer: BulkConsumer; } /// A type that can be used to decode from a bytestring *encoded relative to `Self`*. @@ -37,36 +51,24 @@ pub trait RelativeDecoder { // Path <> Path -impl - RelativeEncoding, PathRc> -{ - pub fn new(path: PathRc, reference: PathRc) -> Self { - RelativeEncoding { - subject: path, - reference, - } - } -} - -impl Encoder - for RelativeEncoding, PathRc> +impl RelativeEncoder> + for PathRc { /// Encode a path relative to another path. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_path_relative) - async fn encode( + async fn relative_encode( &self, + reference: &PathRc, consumer: &mut Consumer, - ) -> Result<(), super::error::EncodingConsumerError> + ) -> Result<(), RelativeEncodeError> where Consumer: BulkConsumer, { - let lcp = self.subject.longest_common_prefix(&self.reference); + let lcp = self.longest_common_prefix(reference); encode_max_power(lcp.component_count(), MCC, consumer).await?; - let suffix = self - .subject - .create_suffix(self.subject.component_count() - lcp.component_count()); + let suffix = self.create_suffix(self.component_count() - lcp.component_count()); suffix.encode(consumer).await?; Ok(()) diff --git a/fuzz/fuzz_targets/path_rel_path_encoding.rs b/fuzz/fuzz_targets/path_rel_path_encoding.rs index a6de5bb..179e42d 100644 --- a/fuzz/fuzz_targets/path_rel_path_encoding.rs +++ b/fuzz/fuzz_targets/path_rel_path_encoding.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; -use willow_data_model::{encoding::relativity::RelativeEncoding, path::PathRc}; +use willow_data_model::path::PathRc; use willow_data_model_fuzz::relative_encoding_roundtrip; const MCL: usize = 16; @@ -16,17 +16,12 @@ fuzz_target!(|data: ( )| { let (path_sub, path_ref, mut consumer) = data; - let relativity = RelativeEncoding { - subject: path_sub, - reference: path_ref, - }; - smol::block_on(async { relative_encoding_roundtrip::< PathRc, PathRc, TestConsumer, - >(relativity, &mut consumer) + >(path_sub, path_ref, &mut consumer) .await; }); }); diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 626edec..e52301a 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -10,7 +10,7 @@ use willow_data_model::{ encoding::{ error::DecodeError, parameters::{Decoder, Encoder}, - relativity::{RelativeDecoder, RelativeEncoding}, + relativity::{RelativeDecoder, RelativeEncoder}, }, path::*, }; @@ -77,17 +77,17 @@ where } pub async fn relative_encoding_roundtrip( - relativity: RelativeEncoding, + subject: T, + reference: R, consumer: &mut TestConsumer, ) where - T: std::fmt::Debug + PartialEq + Eq, + T: std::fmt::Debug + PartialEq + Eq + RelativeEncoder, R: RelativeDecoder, C: BulkConsumer, - RelativeEncoding: Encoder, { let consumer_should_error = consumer.should_error(); - if let Err(_err) = relativity.encode(consumer).await { + if let Err(_err) = subject.relative_encode(&reference, consumer).await { assert!(consumer_should_error); return; } @@ -105,20 +105,15 @@ pub async fn relative_encoding_roundtrip( let mut producer = FromVec::new(new_vec); // Check for correct errors - let decoded_item = relativity - .reference - .relative_decode(&mut producer) - .await - .unwrap(); + let decoded_item = reference.relative_decode(&mut producer).await.unwrap(); - assert_eq!(decoded_item, relativity.subject); + assert_eq!(decoded_item, subject); } pub async fn relative_encoding_random(reference: R, data: &[u8]) where - T: std::fmt::Debug, + T: std::fmt::Debug + RelativeEncoder, R: RelativeDecoder, - RelativeEncoding: Encoder, { let mut producer = SliceProducer::new(data); @@ -128,12 +123,9 @@ where // Can we turn it back into the same encoding? let mut consumer = IntoVec::::new(); - let relativity = RelativeEncoding { - subject: item, - reference, - }; - - relativity.encode(&mut consumer).await.unwrap(); + item.relative_encode(&reference, &mut consumer) + .await + .unwrap(); let encoded = consumer.as_ref().as_slice(); From a977f64affe015cf54724917d7fe3be1169798bd Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 14 Jul 2024 22:47:56 +0100 Subject: [PATCH 03/40] Entry <> Entry encoding --- data-model/src/encoding/compact_width.rs | 25 ++- data-model/src/encoding/relativity.rs | 148 +++++++++++++++++- earthstar/src/cinn25519.rs | 7 +- fuzz/Cargo.toml | 7 + fuzz/fuzz_targets/entry_rel_entry_encoding.rs | 60 +++++++ 5 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 fuzz/fuzz_targets/entry_rel_entry_encoding.rs diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index b768cc3..372e597 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -88,16 +88,33 @@ impl CompactWidth { CompactWidth::Eight => 8, } } + + /// Encode a [`CompactWidth`] as a 2-bit integer `n` such that 2^n gives the bytewidth of the [`CompactWidth`], and then place that 2-bit number into a `u8` at the bit-index of `position`. + pub fn bitmask(&self, position: u8) -> u8 { + let og = match self { + CompactWidth::One => 0b0000_0000, + CompactWidth::Two => 0b0100_0000, + CompactWidth::Four => 0b1000_0000, + CompactWidth::Eight => 0b1100_0000, + }; + + og >> position + } + + pub fn from_2bit_int(n: u8, position: u8) -> Self { + let mask = 0b0000_0011; + let two_bit_int = n >> (6 - position) & mask; + + // Because we sanitise the input down to a 2-bit integer, we can safely unwrap this. + CompactWidth::new(2u8.pow(two_bit_int as u32)).unwrap() + } } /// Encode a `u64` integer as a `compact_width(value)`-byte big-endian integer, and consume that with a [`BulkConsumer`]. pub async fn encode_compact_width_be>( value: u64, consumer: &mut Consumer, -) -> Result<(), EncodingConsumerError> -where - Consumer::Error: Error, -{ +) -> Result<(), EncodingConsumerError> { let width = CompactWidth::from_u64(value).width(); consumer diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index bc89606..211c867 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -1,12 +1,16 @@ -use std::future::Future; +use core::error::Error; +use core::future::Future; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ + compact_width::{decode_compact_width_be, encode_compact_width_be, CompactWidth}, error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, parameters::{Decoder, Encoder}, }, + entry::Entry, + parameters::{NamespaceId, PayloadDigest, SubspaceId}, path::{Path, PathRc}, }; @@ -37,7 +41,7 @@ pub trait RelativeEncoder { Consumer: BulkConsumer; } -/// A type that can be used to decode from a bytestring *encoded relative to `Self`*. +/// A type that can be used to decode `T` from a bytestring *encoded relative to `Self`*. pub trait RelativeDecoder { /// A function from the set of bytestrings *encoded relative to `Self`* to the set of `T` in relation to `Self`. fn relative_decode( @@ -95,9 +99,6 @@ impl RelativeDecoder::decode(producer).await?; @@ -119,3 +120,140 @@ impl RelativeDecoder Entry + +impl + RelativeEncoder, PD>> + for Entry, PD> +where + N: NamespaceId + Encoder, + S: SubspaceId + Encoder, + PD: PayloadDigest + Encoder, +{ + /// Encode an [`Entry`] relative to a reference [`Entry`]. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_etry_relative_entry). + async fn relative_encode( + &self, + reference: &Entry, PD>, + consumer: &mut Consumer, + ) -> Result<(), RelativeEncodeError> + where + Consumer: BulkConsumer, + { + let time_diff = self.timestamp.abs_diff(reference.timestamp); + + let mut header: u8 = 0b0000_0000; + + if self.namespace_id != reference.namespace_id { + header |= 0b1000_0000; + } + + if self.subspace_id != reference.subspace_id { + header |= 0b0100_0000; + } + + if time_diff > 0 && self.timestamp > reference.timestamp { + header |= 0b0010_0000; + } + + header |= CompactWidth::from_u64(time_diff).bitmask(4); + + header |= CompactWidth::from_u64(self.payload_length).bitmask(6); + + if let Err(err) = consumer.consume(header).await { + return Err(RelativeEncodeError::Consumer(EncodingConsumerError { + bytes_consumed: 0, + reason: err, + })); + }; + + if self.namespace_id != reference.namespace_id { + self.namespace_id.encode(consumer).await?; + } + + if self.subspace_id != reference.subspace_id { + self.subspace_id.encode(consumer).await?; + } + + self.path.relative_encode(&reference.path, consumer).await?; + + encode_compact_width_be(time_diff, consumer).await?; + + encode_compact_width_be(self.payload_length, consumer).await?; + + self.payload_digest.encode(consumer).await?; + + Ok(()) + } +} + +impl + RelativeDecoder, PD>> + for Entry, PD> +where + N: NamespaceId + Decoder + std::fmt::Debug, + S: SubspaceId + Decoder + std::fmt::Debug, + PD: PayloadDigest + Decoder, +{ + /// Decode an [`Entry`] relative from this [`Entry`]. + async fn relative_decode( + &self, + producer: &mut Producer, + ) -> Result, PD>, DecodeError> + where + Producer: BulkProducer, + Self: Sized, + { + let mut header_slice = [0u8]; + + producer + .bulk_overwrite_full_slice(&mut header_slice) + .await?; + + let header = header_slice[0]; + + let is_namespace_encoded = header & 0b1000_0000 == 0b1000_0000; + let is_subspace_encoded = header & 0b0100_0000 == 0b0100_0000; + let add_or_subtract_time_diff = header & 0b0010_0000 == 0b0010_0000; + let compact_width_time_diff = CompactWidth::from_2bit_int(header, 4); + let compact_width_payload_length = CompactWidth::from_2bit_int(header, 6); + + let namespace_id = if is_namespace_encoded { + N::decode(producer).await? + } else { + self.namespace_id.clone() + }; + + let subspace_id = if is_subspace_encoded { + S::decode(producer).await? + } else { + self.subspace_id.clone() + }; + + let path = self.path.relative_decode(producer).await?; + + let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; + + let timestamp = if add_or_subtract_time_diff { + self.timestamp + time_diff + } else { + self.timestamp - time_diff + }; + + let payload_length = + decode_compact_width_be(compact_width_payload_length, producer).await?; + + let payload_digest = PD::decode(producer).await?; + + Ok(Entry { + namespace_id, + subspace_id, + path, + timestamp, + payload_length, + payload_digest, + }) + } +} diff --git a/earthstar/src/cinn25519.rs b/earthstar/src/cinn25519.rs index 9a3d531..403317c 100644 --- a/earthstar/src/cinn25519.rs +++ b/earthstar/src/cinn25519.rs @@ -48,9 +48,12 @@ impl<'a, const MIN_LENGTH: usize, const MAX_LENGTH: usize> Arbitrary<'a> for Shortname { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let shortname_string: String = Arbitrary::arbitrary(u)?; + let shortname_bytes: [u8; MIN_LENGTH] = Arbitrary::arbitrary(u)?; - Self::new(&shortname_string).map_err(|_| ArbitraryError::IncorrectFormat) + let shortname = + std::str::from_utf8(&shortname_bytes).map_err(|_| ArbitraryError::IncorrectFormat)?; + + Self::new(shortname).map_err(|_| ArbitraryError::IncorrectFormat) } fn size_hint(_depth: usize) -> (usize, Option) { diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index c3ff599..261a477 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -159,3 +159,10 @@ path = "fuzz_targets/path_rel_path_encoding_random.rs" test = false doc = false bench = false + +[[bin]] +name = "entry_rel_entry_encoding" +path = "fuzz_targets/entry_rel_entry_encoding.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs new file mode 100644 index 0000000..1a7b587 --- /dev/null +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs @@ -0,0 +1,60 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; +use willow_data_model::encoding::parameters::{Decoder, Encoder}; +use willow_data_model::entry::Entry; +use willow_data_model::parameters::PayloadDigest; +use willow_data_model::path::PathRc; +use willow_data_model_fuzz::relative_encoding_roundtrip; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encoder for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decoder for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} + +fuzz_target!(|data: ( + Entry, FakePayloadDigest>, + Entry, FakePayloadDigest>, + TestConsumer +)| { + let (entry_sub, entry_ref, mut consumer) = data; + + smol::block_on(async { + relative_encoding_roundtrip::< + Entry, FakePayloadDigest>, + Entry, FakePayloadDigest>, + TestConsumer, + >(entry_sub, entry_ref, &mut consumer) + .await; + }); +}); From 390db12cda1d111b0980af8c838d5838a5c2f4da Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 15 Jul 2024 08:21:49 +0100 Subject: [PATCH 04/40] Make entry <> entry encoding more robust --- data-model/src/encoding/compact_width.rs | 2 - data-model/src/encoding/relativity.rs | 45 +++++++++++++-- fuzz/Cargo.toml | 7 +++ .../entry_rel_entry_encoding_random.rs | 57 +++++++++++++++++++ fuzz/src/lib.rs | 4 +- 5 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index 372e597..8ec3873 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -1,5 +1,3 @@ -use core::error::Error; - use crate::encoding::error::{DecodeError, EncodingConsumerError}; use crate::encoding::parameters::Decoder; use crate::encoding::unsigned_int::{U16BE, U32BE, U64BE, U8BE}; diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 211c867..562e8b9 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -1,4 +1,3 @@ -use core::error::Error; use core::future::Future; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; @@ -154,7 +153,7 @@ where header |= 0b0100_0000; } - if time_diff > 0 && self.timestamp > reference.timestamp { + if self.timestamp > reference.timestamp { header |= 0b0010_0000; } @@ -214,6 +213,11 @@ where let header = header_slice[0]; + // Verify that bit 3 is 0 as specified - good indicator of invalid data. + if header | 0b1110_1111 == 0b1111_1111 { + return Err(DecodeError::InvalidInput); + } + let is_namespace_encoded = header & 0b1000_0000 == 0b1000_0000; let is_subspace_encoded = header & 0b0100_0000 == 0b0100_0000; let add_or_subtract_time_diff = header & 0b0010_0000 == 0b0010_0000; @@ -226,25 +230,58 @@ where self.namespace_id.clone() }; + // Verify that the encoded namespace wasn't the same as ours + // Which would indicate invalid input + if is_namespace_encoded && namespace_id == self.namespace_id { + return Err(DecodeError::InvalidInput); + } + let subspace_id = if is_subspace_encoded { S::decode(producer).await? } else { self.subspace_id.clone() }; + // Verify that the encoded subspace wasn't the same as ours + // Which would indicate invalid input + if is_subspace_encoded && subspace_id == self.subspace_id { + return Err(DecodeError::InvalidInput); + } + let path = self.path.relative_decode(producer).await?; let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; + // Verify that the compact width of the time diff matches the time diff we just decoded. + if CompactWidth::from_2bit_int(header, 4) != CompactWidth::from_u64(time_diff) { + return Err(DecodeError::InvalidInput); + } + + // Add or subtract safely here to avoid overflows caused by malicious or faulty encodings. let timestamp = if add_or_subtract_time_diff { - self.timestamp + time_diff + self.timestamp + .checked_add(time_diff) + .ok_or(DecodeError::InvalidInput)? } else { - self.timestamp - time_diff + self.timestamp + .checked_sub(time_diff) + .ok_or(DecodeError::InvalidInput)? }; + // Verify that the correct add_or_subtract_time_diff flag was set. + let should_have_subtracted = timestamp <= self.timestamp; + if add_or_subtract_time_diff && should_have_subtracted { + return Err(DecodeError::InvalidInput); + } + let payload_length = decode_compact_width_be(compact_width_payload_length, producer).await?; + // Verify that the compact width of the payload length matches the payload length we just decoded. + if CompactWidth::from_2bit_int(header, 6) != CompactWidth::from_u64(payload_length) { + return Err(DecodeError::InvalidInput); + } + let payload_digest = PD::decode(producer).await?; Ok(Entry { diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 261a477..de4ef3b 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -166,3 +166,10 @@ path = "fuzz_targets/entry_rel_entry_encoding.rs" test = false doc = false bench = false + +[[bin]] +name = "entry_rel_entry_encoding_random" +path = "fuzz_targets/entry_rel_entry_encoding_random.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs new file mode 100644 index 0000000..58877fa --- /dev/null +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs @@ -0,0 +1,57 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; +use willow_data_model::encoding::parameters::{Decoder, Encoder}; +use willow_data_model::entry::Entry; +use willow_data_model::parameters::PayloadDigest; +use willow_data_model::path::PathRc; +use willow_data_model_fuzz::relative_encoding_random; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encoder for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decoder for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} + +fuzz_target!(|data: ( + &[u8], + Entry, FakePayloadDigest>, +)| { + let (random_bytes, ref_entry) = data; + + smol::block_on(async { + relative_encoding_random::< + Entry, FakePayloadDigest>, + Entry, FakePayloadDigest>, + >(ref_entry, random_bytes) + .await; + }); +}); diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index e52301a..5c87951 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -112,8 +112,8 @@ pub async fn relative_encoding_roundtrip( pub async fn relative_encoding_random(reference: R, data: &[u8]) where - T: std::fmt::Debug + RelativeEncoder, - R: RelativeDecoder, + T: RelativeEncoder + std::fmt::Debug, + R: RelativeDecoder + std::fmt::Debug, { let mut producer = SliceProducer::new(data); From 196ce78593fcd0329063c6d1439180d036a36a32 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 15 Jul 2024 08:24:44 +0100 Subject: [PATCH 05/40] Remove RelativeEncodeError --- data-model/src/encoding/relativity.rs | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 562e8b9..b3a39c5 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -13,21 +13,6 @@ use crate::{ path::{Path, PathRc}, }; -/// Returned when a relative encoding fails -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum RelativeEncodeError { - /// The subject could not be encoded relative to the reference (e.g. because it was not logically included by the reference). - IllegalRelativity(), - /// The encoding failed to be consumed by a [`ufotofu::local_nb::Consumer`]. - Consumer(EncodingConsumerError), -} - -impl From> for RelativeEncodeError { - fn from(err: EncodingConsumerError) -> Self { - RelativeEncodeError::Consumer(err) - } -} - /// A type that can be used to encoded to a bytestring *encoded relative to `R`*. pub trait RelativeEncoder { /// A function from the set `Self` to the set of bytestrings *encoded relative to `reference`*. @@ -35,7 +20,7 @@ pub trait RelativeEncoder { &self, reference: &R, consumer: &mut Consumer, - ) -> impl Future>> + ) -> impl Future>> where Consumer: BulkConsumer; } @@ -64,7 +49,7 @@ impl RelativeEncoder, consumer: &mut Consumer, - ) -> Result<(), RelativeEncodeError> + ) -> Result<(), EncodingConsumerError> where Consumer: BulkConsumer, { @@ -137,7 +122,7 @@ where &self, reference: &Entry, PD>, consumer: &mut Consumer, - ) -> Result<(), RelativeEncodeError> + ) -> Result<(), EncodingConsumerError> where Consumer: BulkConsumer, { @@ -162,10 +147,10 @@ where header |= CompactWidth::from_u64(self.payload_length).bitmask(6); if let Err(err) = consumer.consume(header).await { - return Err(RelativeEncodeError::Consumer(EncodingConsumerError { + return Err(EncodingConsumerError { bytes_consumed: 0, reason: err, - })); + }); }; if self.namespace_id != reference.namespace_id { From 73a99636da831ecf3c08e9ed41c324b2e089f70d Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 19 Jul 2024 13:44:43 +0100 Subject: [PATCH 06/40] CompactWdith::decode_fixed_width_bitmask --- data-model/src/encoding/compact_width.rs | 6 +++--- data-model/src/encoding/relativity.rs | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index 8ec3873..8ea5653 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -99,9 +99,9 @@ impl CompactWidth { og >> position } - pub fn from_2bit_int(n: u8, position: u8) -> Self { - let mask = 0b0000_0011; - let two_bit_int = n >> (6 - position) & mask; + pub fn decode_fixed_width_bitmask(mask: u8, offset: u8) -> Self { + let twobit_mask = 0b0000_0011; + let two_bit_int = mask >> (6 - offset) & twobit_mask; // Because we sanitise the input down to a 2-bit integer, we can safely unwrap this. CompactWidth::new(2u8.pow(two_bit_int as u32)).unwrap() diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index b3a39c5..ea11953 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -206,8 +206,8 @@ where let is_namespace_encoded = header & 0b1000_0000 == 0b1000_0000; let is_subspace_encoded = header & 0b0100_0000 == 0b0100_0000; let add_or_subtract_time_diff = header & 0b0010_0000 == 0b0010_0000; - let compact_width_time_diff = CompactWidth::from_2bit_int(header, 4); - let compact_width_payload_length = CompactWidth::from_2bit_int(header, 6); + let compact_width_time_diff = CompactWidth::decode_fixed_width_bitmask(header, 4); + let compact_width_payload_length = CompactWidth::decode_fixed_width_bitmask(header, 6); let namespace_id = if is_namespace_encoded { N::decode(producer).await? @@ -238,7 +238,8 @@ where let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; // Verify that the compact width of the time diff matches the time diff we just decoded. - if CompactWidth::from_2bit_int(header, 4) != CompactWidth::from_u64(time_diff) { + if CompactWidth::decode_fixed_width_bitmask(header, 4) != CompactWidth::from_u64(time_diff) + { return Err(DecodeError::InvalidInput); } @@ -263,7 +264,9 @@ where decode_compact_width_be(compact_width_payload_length, producer).await?; // Verify that the compact width of the payload length matches the payload length we just decoded. - if CompactWidth::from_2bit_int(header, 6) != CompactWidth::from_u64(payload_length) { + if CompactWidth::decode_fixed_width_bitmask(header, 6) + != CompactWidth::from_u64(payload_length) + { return Err(DecodeError::InvalidInput); } From 47a409f51b06f452dcc48deb557ae3a3cba2936d Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 19 Jul 2024 13:46:57 +0100 Subject: [PATCH 07/40] Rename Encoder / Decoder traits --- data-model/src/encoding/compact_width.rs | 2 +- data-model/src/encoding/parameters.rs | 4 ++-- data-model/src/encoding/relativity.rs | 14 ++++++------ data-model/src/encoding/unsigned_int.rs | 18 +++++++-------- data-model/src/entry.rs | 22 +++++++++---------- data-model/src/path.rs | 6 ++--- earthstar/src/cinn25519.rs | 6 ++--- earthstar/src/identity_id.rs | 6 ++--- earthstar/src/namespace_id.rs | 6 ++--- fuzz/fuzz_targets/entry_encoding.rs | 6 ++--- fuzz/fuzz_targets/entry_encoding_random.rs | 6 ++--- fuzz/fuzz_targets/entry_rel_entry_encoding.rs | 6 ++--- .../entry_rel_entry_encoding_random.rs | 6 ++--- fuzz/src/lib.rs | 6 ++--- 14 files changed, 57 insertions(+), 57 deletions(-) diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index 8ea5653..3df8236 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -1,5 +1,5 @@ use crate::encoding::error::{DecodeError, EncodingConsumerError}; -use crate::encoding::parameters::Decoder; +use crate::encoding::parameters::Decodable; use crate::encoding::unsigned_int::{U16BE, U32BE, U64BE, U8BE}; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; diff --git a/data-model/src/encoding/parameters.rs b/data-model/src/encoding/parameters.rs index a5cfa0a..9d93e1f 100644 --- a/data-model/src/encoding/parameters.rs +++ b/data-model/src/encoding/parameters.rs @@ -6,7 +6,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; /// A type that can be encoded to a bytestring, ensuring that any value of `Self` maps to exactly one bytestring. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#encodings_what) -pub trait Encoder { +pub trait Encodable { /// A function from the set `Self` to the set of bytestrings. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#encode_s) @@ -21,7 +21,7 @@ pub trait Encoder { /// A type that can be decoded from a bytestring, ensuring that every valid encoding maps to exactly one member of `Self`. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#encodings_what) -pub trait Decoder { +pub trait Decodable { /// A function from the set of bytestrings to the set of `T`. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#decode_s) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index ea11953..d44323d 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -6,7 +6,7 @@ use crate::{ compact_width::{decode_compact_width_be, encode_compact_width_be, CompactWidth}, error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, }, entry::Entry, parameters::{NamespaceId, PayloadDigest, SubspaceId}, @@ -111,9 +111,9 @@ impl RelativeEncoder, PD>> for Entry, PD> where - N: NamespaceId + Encoder, - S: SubspaceId + Encoder, - PD: PayloadDigest + Encoder, + N: NamespaceId + Encodable, + S: SubspaceId + Encodable, + PD: PayloadDigest + Encodable, { /// Encode an [`Entry`] relative to a reference [`Entry`]. /// @@ -177,9 +177,9 @@ impl RelativeDecoder, PD>> for Entry, PD> where - N: NamespaceId + Decoder + std::fmt::Debug, - S: SubspaceId + Decoder + std::fmt::Debug, - PD: PayloadDigest + Decoder, + N: NamespaceId + Decodable + std::fmt::Debug, + S: SubspaceId + Decodable + std::fmt::Debug, + PD: PayloadDigest + Decodable, { /// Decode an [`Entry`] relative from this [`Entry`]. async fn relative_decode( diff --git a/data-model/src/encoding/unsigned_int.rs b/data-model/src/encoding/unsigned_int.rs index e4e35f3..7be4c30 100644 --- a/data-model/src/encoding/unsigned_int.rs +++ b/data-model/src/encoding/unsigned_int.rs @@ -2,13 +2,13 @@ use core::mem::size_of; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::encoding::error::{DecodeError, EncodingConsumerError}; -use crate::encoding::parameters::{Decoder, Encoder}; +use crate::encoding::parameters::{Decodable, Encodable}; /// A `u8` wrapper that implements [`Encoding`] and [`Decoding`] by encoding as a big-endian fixed-width integer. #[derive(PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct U8BE(u8); -impl Encoder for U8BE { +impl Encodable for U8BE { async fn encode( &self, consumer: &mut Consumer, @@ -22,7 +22,7 @@ impl Encoder for U8BE { } } -impl Decoder for U8BE { +impl Decodable for U8BE { async fn decode(producer: &mut Producer) -> Result> where Producer: BulkProducer, @@ -50,7 +50,7 @@ impl From for u64 { #[derive(PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct U16BE(u16); -impl Encoder for U16BE { +impl Encodable for U16BE { async fn encode( &self, consumer: &mut Consumer, @@ -64,7 +64,7 @@ impl Encoder for U16BE { } } -impl Decoder for U16BE { +impl Decodable for U16BE { async fn decode(producer: &mut Producer) -> Result> where Producer: BulkProducer, @@ -92,7 +92,7 @@ impl From for u64 { #[derive(PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct U32BE(u32); -impl Encoder for U32BE { +impl Encodable for U32BE { async fn encode( &self, consumer: &mut Consumer, @@ -106,7 +106,7 @@ impl Encoder for U32BE { } } -impl Decoder for U32BE { +impl Decodable for U32BE { async fn decode(producer: &mut Producer) -> Result> where Producer: BulkProducer, @@ -134,7 +134,7 @@ impl From for u64 { #[derive(PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct U64BE(u64); -impl Encoder for U64BE { +impl Encodable for U64BE { async fn encode( &self, consumer: &mut Consumer, @@ -148,7 +148,7 @@ impl Encoder for U64BE { } } -impl Decoder for U64BE { +impl Decodable for U64BE { async fn decode(producer: &mut Producer) -> Result> where Producer: BulkProducer, diff --git a/data-model/src/entry.rs b/data-model/src/entry.rs index d867a53..86a11f3 100644 --- a/data-model/src/entry.rs +++ b/data-model/src/entry.rs @@ -7,7 +7,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ error::{DecodeError, EncodingConsumerError}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, unsigned_int::U64BE, }, parameters::{IsAuthorisedWrite, NamespaceId, PayloadDigest, SubspaceId}, @@ -69,12 +69,12 @@ where } } -impl Encoder for Entry +impl Encodable for Entry where - N: NamespaceId + Encoder, - S: SubspaceId + Encoder, - P: Path + Encoder, - PD: PayloadDigest + Encoder, + N: NamespaceId + Encodable, + S: SubspaceId + Encodable, + P: Path + Encodable, + PD: PayloadDigest + Encodable, { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError<::Error>> where @@ -93,12 +93,12 @@ where } } -impl Decoder for Entry +impl Decodable for Entry where - N: NamespaceId + Decoder, - S: SubspaceId + Decoder, - P: Path + Decoder, - PD: PayloadDigest + Decoder, + N: NamespaceId + Decodable, + S: SubspaceId + Decodable, + P: Path + Decodable, + PD: PayloadDigest + Decodable, { async fn decode(producer: &mut Prod) -> Result> where diff --git a/data-model/src/path.rs b/data-model/src/path.rs index 41b9977..e80ae59 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -6,7 +6,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::encoding::{ error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, }; #[derive(Debug)] @@ -422,7 +422,7 @@ impl PartialOrd for PathRc } } -impl Encoder for PathRc { +impl Encodable for PathRc { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -439,7 +439,7 @@ impl Encoder for PathRc Decoder for PathRc { +impl Decodable for PathRc { async fn decode

(producer: &mut P) -> Result> where P: BulkProducer, diff --git a/earthstar/src/cinn25519.rs b/earthstar/src/cinn25519.rs index 403317c..aef9100 100644 --- a/earthstar/src/cinn25519.rs +++ b/earthstar/src/cinn25519.rs @@ -3,7 +3,7 @@ use either::Either; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::encoding::{ error::{DecodeError, EncodingConsumerError}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, }; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -71,7 +71,7 @@ impl Cinn25519PublicKey Encoder +impl Encodable for Cinn25519PublicKey { async fn encode( @@ -102,7 +102,7 @@ impl Encoder } } -impl Decoder +impl Decodable for Cinn25519PublicKey { async fn decode( diff --git a/earthstar/src/identity_id.rs b/earthstar/src/identity_id.rs index 435518d..d90d7ca 100644 --- a/earthstar/src/identity_id.rs +++ b/earthstar/src/identity_id.rs @@ -3,7 +3,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::{ encoding::{ error::{DecodeError, EncodingConsumerError}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, }, parameters::SubspaceId, }; @@ -22,7 +22,7 @@ impl Default for IdentityIdentifier { } } -impl Encoder for IdentityIdentifier { +impl Encodable for IdentityIdentifier { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -32,7 +32,7 @@ impl Encoder for IdentityIdentifier { } } -impl Decoder for IdentityIdentifier { +impl Decodable for IdentityIdentifier { async fn decode

(producer: &mut P) -> Result> where P: BulkProducer, diff --git a/earthstar/src/namespace_id.rs b/earthstar/src/namespace_id.rs index 27da503..b4e5da0 100644 --- a/earthstar/src/namespace_id.rs +++ b/earthstar/src/namespace_id.rs @@ -3,7 +3,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::{ encoding::{ error::{DecodeError, EncodingConsumerError}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, }, parameters::NamespaceId, }; @@ -22,7 +22,7 @@ impl Default for NamespaceIdentifier { } } -impl Encoder for NamespaceIdentifier { +impl Encodable for NamespaceIdentifier { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -32,7 +32,7 @@ impl Encoder for NamespaceIdentifier { } } -impl Decoder for NamespaceIdentifier { +impl Decodable for NamespaceIdentifier { async fn decode

(producer: &mut P) -> Result::Error>> where P: BulkProducer, diff --git a/fuzz/fuzz_targets/entry_encoding.rs b/fuzz/fuzz_targets/entry_encoding.rs index ad84e96..6c50c84 100644 --- a/fuzz/fuzz_targets/entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_encoding.rs @@ -7,7 +7,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decoder, Encoder}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; use willow_data_model::path::PathRc; @@ -16,7 +16,7 @@ use willow_data_model_fuzz::encoding_roundtrip; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); -impl Encoder for FakePayloadDigest { +impl Encodable for FakePayloadDigest { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -27,7 +27,7 @@ impl Encoder for FakePayloadDigest { } } -impl Decoder for FakePayloadDigest { +impl Decodable for FakePayloadDigest { async fn decode

(producer: &mut P) -> Result> where P: BulkProducer, diff --git a/fuzz/fuzz_targets/entry_encoding_random.rs b/fuzz/fuzz_targets/entry_encoding_random.rs index 972fe5d..c3362a0 100644 --- a/fuzz/fuzz_targets/entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_encoding_random.rs @@ -9,7 +9,7 @@ use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; use willow_data_model::{ encoding::{ error::{DecodeError, EncodingConsumerError}, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, }, entry::Entry, parameters::PayloadDigest, @@ -19,7 +19,7 @@ use willow_data_model::{ #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); -impl Encoder for FakePayloadDigest { +impl Encodable for FakePayloadDigest { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -30,7 +30,7 @@ impl Encoder for FakePayloadDigest { } } -impl Decoder for FakePayloadDigest { +impl Decodable for FakePayloadDigest { async fn decode

(producer: &mut P) -> Result> where P: BulkProducer, diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs index 1a7b587..a0a4fdf 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs @@ -7,7 +7,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decoder, Encoder}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; use willow_data_model::path::PathRc; @@ -16,7 +16,7 @@ use willow_data_model_fuzz::relative_encoding_roundtrip; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); -impl Encoder for FakePayloadDigest { +impl Encodable for FakePayloadDigest { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -27,7 +27,7 @@ impl Encoder for FakePayloadDigest { } } -impl Decoder for FakePayloadDigest { +impl Decodable for FakePayloadDigest { async fn decode

(producer: &mut P) -> Result> where P: BulkProducer, diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs index 58877fa..894138b 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs @@ -6,7 +6,7 @@ use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decoder, Encoder}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; use willow_data_model::path::PathRc; @@ -15,7 +15,7 @@ use willow_data_model_fuzz::relative_encoding_random; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); -impl Encoder for FakePayloadDigest { +impl Encodable for FakePayloadDigest { async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> where C: BulkConsumer, @@ -26,7 +26,7 @@ impl Encoder for FakePayloadDigest { } } -impl Decoder for FakePayloadDigest { +impl Decodable for FakePayloadDigest { async fn decode

(producer: &mut P) -> Result> where P: BulkProducer, diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 5c87951..da203e0 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -9,7 +9,7 @@ use ufotofu::{ use willow_data_model::{ encoding::{ error::DecodeError, - parameters::{Decoder, Encoder}, + parameters::{Decodable, Encodable}, relativity::{RelativeDecoder, RelativeEncoder}, }, path::*, @@ -17,7 +17,7 @@ use willow_data_model::{ pub async fn encoding_roundtrip(item: T, consumer: &mut TestConsumer) where - T: Encoder + Decoder + std::fmt::Debug + PartialEq + Eq, + T: Encodable + Decodable + std::fmt::Debug + PartialEq + Eq, C: BulkConsumer, { let consumer_should_error = consumer.should_error(); @@ -47,7 +47,7 @@ where pub async fn encoding_random(data: &[u8]) where - T: Encoder + Decoder, + T: Encodable + Decodable, { let mut producer = SliceProducer::new(data); From 5505040d759ba78ddc67a5e8a1b05232a779e7b4 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 19 Jul 2024 14:13:28 +0100 Subject: [PATCH 08/40] Refactor RelativeDecodable (and rename) --- data-model/src/encoding/relativity.rs | 44 ++++++++++++++------------- fuzz/src/lib.rs | 14 ++++----- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index d44323d..201df57 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -14,7 +14,7 @@ use crate::{ }; /// A type that can be used to encoded to a bytestring *encoded relative to `R`*. -pub trait RelativeEncoder { +pub trait RelativeEncodable { /// A function from the set `Self` to the set of bytestrings *encoded relative to `reference`*. fn relative_encode( &self, @@ -26,12 +26,12 @@ pub trait RelativeEncoder { } /// A type that can be used to decode `T` from a bytestring *encoded relative to `Self`*. -pub trait RelativeDecoder { +pub trait RelativeDecodable { /// A function from the set of bytestrings *encoded relative to `Self`* to the set of `T` in relation to `Self`. fn relative_decode( - &self, + reference: &R, producer: &mut Producer, - ) -> impl Future>> + ) -> impl Future>> where Producer: BulkProducer, Self: Sized; @@ -39,7 +39,7 @@ pub trait RelativeDecoder { // Path <> Path -impl RelativeEncoder> +impl RelativeEncodable> for PathRc { /// Encode a path relative to another path. @@ -63,14 +63,14 @@ impl RelativeEncoder RelativeDecoder> +impl RelativeDecodable> for PathRc { /// Decode a path relative to this path. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_path_relative) async fn relative_decode( - &self, + reference: &PathRc, producer: &mut Producer, ) -> Result> where @@ -79,11 +79,11 @@ impl RelativeDecoder self.component_count() as u64 { + if lcp > reference.component_count() as u64 { return Err(DecodeError::InvalidInput); } - let prefix = self.create_prefix(lcp as usize); + let prefix = reference.create_prefix(lcp as usize); let suffix = PathRc::::decode(producer).await?; let mut new = prefix; @@ -95,7 +95,7 @@ impl RelativeDecoder RelativeDecoder Entry impl - RelativeEncoder, PD>> + RelativeEncodable, PD>> for Entry, PD> where N: NamespaceId + Encodable, @@ -174,7 +174,7 @@ where } impl - RelativeDecoder, PD>> + RelativeDecodable, PD>> for Entry, PD> where N: NamespaceId + Decodable + std::fmt::Debug, @@ -183,7 +183,7 @@ where { /// Decode an [`Entry`] relative from this [`Entry`]. async fn relative_decode( - &self, + reference: &Entry, PD>, producer: &mut Producer, ) -> Result, PD>, DecodeError> where @@ -212,28 +212,28 @@ where let namespace_id = if is_namespace_encoded { N::decode(producer).await? } else { - self.namespace_id.clone() + reference.namespace_id.clone() }; // Verify that the encoded namespace wasn't the same as ours // Which would indicate invalid input - if is_namespace_encoded && namespace_id == self.namespace_id { + if is_namespace_encoded && namespace_id == reference.namespace_id { return Err(DecodeError::InvalidInput); } let subspace_id = if is_subspace_encoded { S::decode(producer).await? } else { - self.subspace_id.clone() + reference.subspace_id.clone() }; // Verify that the encoded subspace wasn't the same as ours // Which would indicate invalid input - if is_subspace_encoded && subspace_id == self.subspace_id { + if is_subspace_encoded && subspace_id == reference.subspace_id { return Err(DecodeError::InvalidInput); } - let path = self.path.relative_decode(producer).await?; + let path = PathRc::::relative_decode(&reference.path, producer).await?; let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; @@ -245,17 +245,19 @@ where // Add or subtract safely here to avoid overflows caused by malicious or faulty encodings. let timestamp = if add_or_subtract_time_diff { - self.timestamp + reference + .timestamp .checked_add(time_diff) .ok_or(DecodeError::InvalidInput)? } else { - self.timestamp + reference + .timestamp .checked_sub(time_diff) .ok_or(DecodeError::InvalidInput)? }; // Verify that the correct add_or_subtract_time_diff flag was set. - let should_have_subtracted = timestamp <= self.timestamp; + let should_have_subtracted = timestamp <= reference.timestamp; if add_or_subtract_time_diff && should_have_subtracted { return Err(DecodeError::InvalidInput); } diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index da203e0..8031bb5 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -10,7 +10,7 @@ use willow_data_model::{ encoding::{ error::DecodeError, parameters::{Decodable, Encodable}, - relativity::{RelativeDecoder, RelativeEncoder}, + relativity::{RelativeDecodable, RelativeEncodable}, }, path::*, }; @@ -81,8 +81,8 @@ pub async fn relative_encoding_roundtrip( reference: R, consumer: &mut TestConsumer, ) where - T: std::fmt::Debug + PartialEq + Eq + RelativeEncoder, - R: RelativeDecoder, + T: std::fmt::Debug + PartialEq + Eq + RelativeEncodable + RelativeDecodable, + R: std::fmt::Debug, C: BulkConsumer, { let consumer_should_error = consumer.should_error(); @@ -105,19 +105,19 @@ pub async fn relative_encoding_roundtrip( let mut producer = FromVec::new(new_vec); // Check for correct errors - let decoded_item = reference.relative_decode(&mut producer).await.unwrap(); + let decoded_item = T::relative_decode(&reference, &mut producer).await.unwrap(); assert_eq!(decoded_item, subject); } pub async fn relative_encoding_random(reference: R, data: &[u8]) where - T: RelativeEncoder + std::fmt::Debug, - R: RelativeDecoder + std::fmt::Debug, + T: RelativeEncodable + RelativeDecodable + std::fmt::Debug, + R: std::fmt::Debug, { let mut producer = SliceProducer::new(data); - match reference.relative_decode(&mut producer).await { + match T::relative_decode(&reference, &mut producer).await { Ok(item) => { // It decoded to a valid item! Gasp! // Can we turn it back into the same encoding? From 622015196dda1e02e3c9af238ca61a6681380ddb Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 19 Jul 2024 15:14:14 +0100 Subject: [PATCH 09/40] Add consume_byte and produce_byte helpers --- data-model/src/encoding/bytes.rs | 37 +++++++++++++++++++++++++++ data-model/src/encoding/mod.rs | 1 + data-model/src/encoding/relativity.rs | 16 +++--------- 3 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 data-model/src/encoding/bytes.rs diff --git a/data-model/src/encoding/bytes.rs b/data-model/src/encoding/bytes.rs new file mode 100644 index 0000000..dfde72e --- /dev/null +++ b/data-model/src/encoding/bytes.rs @@ -0,0 +1,37 @@ +use either::Either; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; + +use crate::encoding::error::{DecodeError, EncodingConsumerError}; + +/// Have `Consumer` consume a single byte, or return an [`EncodingConsumerError`] with its `bytes_consumed` field set to `consumed_so_far`. +pub async fn consume_byte( + byte: u8, + consumed_so_far: usize, + consumer: &mut Consumer, +) -> Result<(), EncodingConsumerError> +where + Consumer: BulkConsumer, +{ + if let Err(err) = consumer.consume(byte).await { + return Err(EncodingConsumerError { + bytes_consumed: consumed_so_far, + reason: err, + }); + }; + + Ok(()) +} + +/// Have `Producer` produce a single byte, or return an error if the final value was produced or the producer experienced an error. +pub async fn produce_byte( + producer: &mut Producer, +) -> Result> +where + Producer: BulkProducer, +{ + match producer.produce().await { + Ok(Either::Left(item)) => Ok(item), + Ok(Either::Right(_)) => Err(DecodeError::InvalidInput), + Err(err) => Err(DecodeError::Producer(err)), + } +} diff --git a/data-model/src/encoding/mod.rs b/data-model/src/encoding/mod.rs index 717bac0..a1c1717 100644 --- a/data-model/src/encoding/mod.rs +++ b/data-model/src/encoding/mod.rs @@ -1,3 +1,4 @@ +pub mod bytes; pub mod compact_width; pub mod error; pub mod max_power; diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 201df57..73883da 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -3,6 +3,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ + bytes::{consume_byte, produce_byte}, compact_width::{decode_compact_width_be, encode_compact_width_be, CompactWidth}, error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, @@ -146,12 +147,7 @@ where header |= CompactWidth::from_u64(self.payload_length).bitmask(6); - if let Err(err) = consumer.consume(header).await { - return Err(EncodingConsumerError { - bytes_consumed: 0, - reason: err, - }); - }; + consume_byte(header, 0, consumer).await?; if self.namespace_id != reference.namespace_id { self.namespace_id.encode(consumer).await?; @@ -190,13 +186,7 @@ where Producer: BulkProducer, Self: Sized, { - let mut header_slice = [0u8]; - - producer - .bulk_overwrite_full_slice(&mut header_slice) - .await?; - - let header = header_slice[0]; + let header = produce_byte(producer).await?; // Verify that bit 3 is 0 as specified - good indicator of invalid data. if header | 0b1110_1111 == 0b1111_1111 { From 737d476a5689fe17c35a947d1e911a98425edfa8 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 19 Jul 2024 16:11:21 +0100 Subject: [PATCH 10/40] Add is_bitflagged utility fn --- data-model/src/encoding/bytes.rs | 16 ++++++++++++++++ data-model/src/encoding/relativity.rs | 12 ++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/data-model/src/encoding/bytes.rs b/data-model/src/encoding/bytes.rs index dfde72e..55898f8 100644 --- a/data-model/src/encoding/bytes.rs +++ b/data-model/src/encoding/bytes.rs @@ -35,3 +35,19 @@ where Err(err) => Err(DecodeError::Producer(err)), } } + +pub fn is_bitflagged(byte: u8, position: u8) -> bool { + let mask = match position { + 0 => 0b1000_0000, + 1 => 0b0100_0000, + 2 => 0b0010_0000, + 3 => 0b0001_0000, + 4 => 0b0000_1000, + 5 => 0b0000_0100, + 6 => 0b0000_0010, + 7 => 0b0000_0001, + _ => panic!("Can't check for a bitflag at a position greater than 7"), + }; + + byte & mask == mask +} diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 73883da..08770e8 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -3,7 +3,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ - bytes::{consume_byte, produce_byte}, + bytes::{consume_byte, is_bitflagged, produce_byte}, compact_width::{decode_compact_width_be, encode_compact_width_be, CompactWidth}, error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, @@ -188,14 +188,14 @@ where { let header = produce_byte(producer).await?; - // Verify that bit 3 is 0 as specified - good indicator of invalid data. - if header | 0b1110_1111 == 0b1111_1111 { + // Verify that bit 3 is 0 as specified. + if is_bitflagged(header, 3) { return Err(DecodeError::InvalidInput); } - let is_namespace_encoded = header & 0b1000_0000 == 0b1000_0000; - let is_subspace_encoded = header & 0b0100_0000 == 0b0100_0000; - let add_or_subtract_time_diff = header & 0b0010_0000 == 0b0010_0000; + let is_namespace_encoded = is_bitflagged(header, 0); + let is_subspace_encoded = is_bitflagged(header, 1); + let add_or_subtract_time_diff = is_bitflagged(header, 2); let compact_width_time_diff = CompactWidth::decode_fixed_width_bitmask(header, 4); let compact_width_payload_length = CompactWidth::decode_fixed_width_bitmask(header, 6); From 4c6150e030edca5f4e54594666f318313e624bf3 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 19 Jul 2024 16:24:05 +0100 Subject: [PATCH 11/40] Better shortname arbitrary --- data-model/src/encoding/relativity.rs | 2 ++ earthstar/src/cinn25519.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 08770e8..0284ad4 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -178,6 +178,8 @@ where PD: PayloadDigest + Decodable, { /// Decode an [`Entry`] relative from this [`Entry`]. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_etry_relative_entry). async fn relative_decode( reference: &Entry, PD>, producer: &mut Producer, diff --git a/earthstar/src/cinn25519.rs b/earthstar/src/cinn25519.rs index aef9100..815370b 100644 --- a/earthstar/src/cinn25519.rs +++ b/earthstar/src/cinn25519.rs @@ -48,7 +48,7 @@ impl<'a, const MIN_LENGTH: usize, const MAX_LENGTH: usize> Arbitrary<'a> for Shortname { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let shortname_bytes: [u8; MIN_LENGTH] = Arbitrary::arbitrary(u)?; + let shortname_bytes: [u8; MAX_LENGTH] = Arbitrary::arbitrary(u)?; let shortname = std::str::from_utf8(&shortname_bytes).map_err(|_| ArbitraryError::IncorrectFormat)?; From 8ce628e516f66c61a1b905bb5499dec070258099 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sat, 20 Jul 2024 10:46:17 +0100 Subject: [PATCH 12/40] Fix merge --- fuzz/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index bc8a90d..c8a460a 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -79,15 +79,11 @@ pub async fn relative_encoding_roundtrip( R: std::fmt::Debug, C: BulkConsumer, { - let consumer_should_error = consumer.should_error(); - if let Err(_err) = subject.relative_encode(&reference, consumer).await { - assert!(consumer_should_error); return; } if let Err(_err) = consumer.flush().await { - assert!(consumer_should_error); return; } @@ -109,7 +105,7 @@ where T: RelativeEncodable + RelativeDecodable + std::fmt::Debug, R: std::fmt::Debug, { - let mut producer = SliceProducer::new(data); + let mut producer = FromSlice::new(data); match T::relative_decode(&reference, &mut producer).await { Ok(item) => { From 1b22e21505e61cdd552c169f461ff8674c3d5989 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sat, 20 Jul 2024 10:59:23 +0100 Subject: [PATCH 13/40] Simplify EncodingConsumerError --- data-model/src/encoding/bytes.rs | 23 ++--------------------- data-model/src/encoding/error.rs | 26 ++++++++++---------------- data-model/src/encoding/relativity.rs | 4 ++-- earthstar/src/cinn25519.rs | 14 +++----------- 4 files changed, 17 insertions(+), 50 deletions(-) diff --git a/data-model/src/encoding/bytes.rs b/data-model/src/encoding/bytes.rs index 55898f8..5092916 100644 --- a/data-model/src/encoding/bytes.rs +++ b/data-model/src/encoding/bytes.rs @@ -1,26 +1,7 @@ use either::Either; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use ufotofu::local_nb::BulkProducer; -use crate::encoding::error::{DecodeError, EncodingConsumerError}; - -/// Have `Consumer` consume a single byte, or return an [`EncodingConsumerError`] with its `bytes_consumed` field set to `consumed_so_far`. -pub async fn consume_byte( - byte: u8, - consumed_so_far: usize, - consumer: &mut Consumer, -) -> Result<(), EncodingConsumerError> -where - Consumer: BulkConsumer, -{ - if let Err(err) = consumer.consume(byte).await { - return Err(EncodingConsumerError { - bytes_consumed: consumed_so_far, - reason: err, - }); - }; - - Ok(()) -} +use crate::encoding::error::DecodeError; /// Have `Producer` produce a single byte, or return an error if the final value was produced or the producer experienced an error. pub async fn produce_byte( diff --git a/data-model/src/encoding/error.rs b/data-model/src/encoding/error.rs index e3a0a56..0d49e57 100644 --- a/data-model/src/encoding/error.rs +++ b/data-model/src/encoding/error.rs @@ -5,19 +5,17 @@ use ufotofu::common::errors::{ConsumeFullSliceError, OverwriteFullSliceError}; /// Returned when a encoding fails to be consumed by a [`ufotofu::local_nb::Consumer`]. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct EncodingConsumerError { - /// The number of bytes which were consumed before the error. - pub bytes_consumed: usize, - /// The error returned on the final and failed attempt to consume bytes. - pub reason: E, -} +pub struct EncodingConsumerError(E); impl From> for EncodingConsumerError { fn from(err: ConsumeFullSliceError) -> Self { - EncodingConsumerError { - bytes_consumed: err.consumed, - reason: err.reason, - } + EncodingConsumerError(err.reason) + } +} + +impl From for EncodingConsumerError { + fn from(value: E) -> Self { + Self(value) } } @@ -26,17 +24,13 @@ where E: 'static + Error, { fn source(&self) -> Option<&(dyn Error + 'static)> { - Some(&self.reason) + Some(&self.0) } } impl Display for EncodingConsumerError { fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { - write!( - f, - "The consumer failed to consume after consuming {} bytes", - self.bytes_consumed - ) + write!(f, "The consumer returned an error after trying to consume",) } } diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 4a37b13..8f8d76f 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -3,7 +3,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ - bytes::{consume_byte, is_bitflagged, produce_byte}, + bytes::{is_bitflagged, produce_byte}, compact_width::{decode_compact_width_be, encode_compact_width_be, CompactWidth}, error::{DecodeError, EncodingConsumerError}, max_power::{decode_max_power, encode_max_power}, @@ -164,7 +164,7 @@ where header |= CompactWidth::from_u64(self.payload_length).bitmask(6); - consume_byte(header, 0, consumer).await?; + consumer.consume(header).await?; if self.namespace_id != reference.namespace_id { self.namespace_id.encode(consumer).await?; diff --git a/earthstar/src/cinn25519.rs b/earthstar/src/cinn25519.rs index 815370b..b57933a 100644 --- a/earthstar/src/cinn25519.rs +++ b/earthstar/src/cinn25519.rs @@ -48,12 +48,9 @@ impl<'a, const MIN_LENGTH: usize, const MAX_LENGTH: usize> Arbitrary<'a> for Shortname { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let shortname_bytes: [u8; MAX_LENGTH] = Arbitrary::arbitrary(u)?; + let shortname_string: String = Arbitrary::arbitrary(u)?; - let shortname = - std::str::from_utf8(&shortname_bytes).map_err(|_| ArbitraryError::IncorrectFormat)?; - - Self::new(shortname).map_err(|_| ArbitraryError::IncorrectFormat) + Self::new(&shortname_string).map_err(|_| ArbitraryError::IncorrectFormat) } fn size_hint(_depth: usize) -> (usize, Option) { @@ -88,12 +85,7 @@ impl Encodable consumer.bulk_consume_full_slice(&self.shortname.0).await?; if MIN_LENGTH < MAX_LENGTH { - if let Err(err) = consumer.consume(0x0).await { - return Err(EncodingConsumerError { - bytes_consumed: self.shortname.0.len(), - reason: err, - }); - } + consumer.consume(0x0).await?; } consumer.bulk_consume_full_slice(&self.underlying).await?; From 71b1f52ed6db69e7afd30599b17637ab37274a07 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sat, 20 Jul 2024 13:39:36 +0100 Subject: [PATCH 14/40] fuzz pairing --- data-model/src/encoding/relativity.rs | 122 ++++++++++++++++++ data-model/src/entry.rs | 22 +++- data-model/src/grouping/area.rs | 47 +++++++ data-model/src/grouping/range.rs | 36 ++++++ fuzz/Cargo.toml | 7 + fuzz/fuzz_targets/entry_rel_entry_encoding.rs | 2 + .../entry_rel_namespace_area_encoding.rs | 69 ++++++++++ 7 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 8f8d76f..0bc1268 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -10,6 +10,7 @@ use crate::{ parameters::{Decodable, Encodable}, }, entry::Entry, + grouping::area::{Area, AreaSubspace}, parameters::{NamespaceId, PayloadDigest, SubspaceId}, path::Path, }; @@ -292,3 +293,124 @@ where }) } } + +impl + RelativeEncodable<(N, Area)> for Entry +where + N: NamespaceId + Encodable, + S: SubspaceId + Encodable, + PD: PayloadDigest + Encodable, +{ + async fn relative_encode( + &self, + reference: &(N, Area), + consumer: &mut Consumer, + ) -> Result<(), EncodingConsumerError> + where + Consumer: BulkConsumer, + { + let (namespace, out) = reference; + + if &self.namespace_id != namespace { + panic!("Tried to encode an entry relative to a namespace it does not belong to") + } + + if out.includes_entry(self) { + panic!("Tried to encode an entry relative to an area it is not included by") + } + + let time_diff = core::cmp::min( + self.timestamp - out.times.start, + u64::from(&out.times.end) - self.timestamp, + ); + + let mut header = 0b0000_0000; + + if out.subspace == AreaSubspace::Any { + header |= 0b1000_0000; + } + + if self.timestamp - out.times.start <= u64::from(&out.times.end) - self.timestamp { + header |= 0b0100_0000; + } + + header |= CompactWidth::from_u64(time_diff).bitmask(2); + header |= CompactWidth::from_u64(self.payload_length).bitmask(4); + + consumer.consume(header).await?; + + if out.subspace != AreaSubspace::Any { + self.subspace_id.encode(consumer).await?; + } + + self.path.relative_encode(&out.path, consumer).await?; + encode_compact_width_be(time_diff, consumer).await?; + encode_compact_width_be(self.payload_length, consumer).await?; + self.payload_digest.encode(consumer).await?; + + Ok(()) + } +} + +impl + RelativeDecodable<(N, Area)> for Entry +where + N: NamespaceId + Decodable, + S: SubspaceId + Decodable, + PD: PayloadDigest + Decodable, +{ + async fn relative_decode( + reference: &(N, Area), + producer: &mut Producer, + ) -> Result> + where + Producer: BulkProducer, + Self: Sized, + { + let (namespace, out) = reference; + + let header = produce_byte(producer).await?; + + let is_subspace_encoded = is_bitflagged(header, 0); + let add_time_diff_to_start = is_bitflagged(header, 1); + let time_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 2); + let payload_length_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 4); + + if is_bitflagged(header, 6) || is_bitflagged(header, 7) { + return Err(DecodeError::InvalidInput); + } + + let subspace_id = if is_subspace_encoded { + S::decode(producer).await? + } else { + match &out.subspace { + AreaSubspace::Any => return Err(DecodeError::InvalidInput), + AreaSubspace::Id(id) => id.clone(), + } + }; + + let path = Path::relative_decode(&out.path, producer).await?; + + let time_diff = decode_compact_width_be(time_diff_compact_width, producer).await?; + let payload_length = + decode_compact_width_be(payload_length_compact_width, producer).await?; + + let payload_digest = PD::decode(producer).await?; + + let timestamp = if add_time_diff_to_start { + out.times.start.checked_add(time_diff) + } else { + u64::from(&out.times.end).checked_sub(time_diff) + } + .ok_or(DecodeError::InvalidInput)?; + + Ok(Self { + namespace_id: namespace.clone(), + subspace_id, + path, + timestamp, + payload_length, + payload_digest, + }) + } +} diff --git a/data-model/src/entry.rs b/data-model/src/entry.rs index dd58ca5..3aee234 100644 --- a/data-model/src/entry.rs +++ b/data-model/src/entry.rs @@ -129,16 +129,29 @@ where PD: PayloadDigest + Arbitrary<'a>, { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let namespace_id: N = Arbitrary::arbitrary(u)?; + + let subspace_id: S = Arbitrary::arbitrary(u)?; + + let path: Path = Arbitrary::arbitrary(u)?; + + // println!("3"); + + let payload_digest: PD = Arbitrary::arbitrary(u)?; + + // println!("4"); + Ok(Self { - namespace_id: Arbitrary::arbitrary(u)?, - subspace_id: Arbitrary::arbitrary(u)?, - path: Arbitrary::arbitrary(u)?, - payload_digest: Arbitrary::arbitrary(u)?, + namespace_id, + subspace_id, + path, + payload_digest, payload_length: Arbitrary::arbitrary(u)?, timestamp: Arbitrary::arbitrary(u)?, }) } + /* fn size_hint(depth: usize) -> (usize, Option) { and_all(&[ N::size_hint(depth), @@ -149,6 +162,7 @@ where u64::size_hint(depth), ]) } + */ } /// An error indicating an [`AuthorisationToken`] does not authorise the writing of this entry. diff --git a/data-model/src/grouping/area.rs b/data-model/src/grouping/area.rs index 36738dc..e82c8d9 100644 --- a/data-model/src/grouping/area.rs +++ b/data-model/src/grouping/area.rs @@ -1,3 +1,5 @@ +use arbitrary::{size_hint::and_all, Arbitrary}; + use crate::{ entry::{Entry, Timestamp}, parameters::{NamespaceId, PayloadDigest, SubspaceId}, @@ -36,6 +38,24 @@ impl AreaSubspace { } } +#[cfg(feature = "dev")] +impl<'a, S> Arbitrary<'a> for AreaSubspace +where + S: SubspaceId + Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let is_any: bool = Arbitrary::arbitrary(u)?; + + if !is_any { + let subspace: S = Arbitrary::arbitrary(u)?; + + return Ok(Self::Id(subspace)); + } + + Ok(Self::Any) + } +} + /// A grouping of entries. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#areas). #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -99,6 +119,33 @@ impl Area Arbitrary<'a> + for Area +where + S: SubspaceId + Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let subspace: AreaSubspace = Arbitrary::arbitrary(u)?; + let path: Path = Arbitrary::arbitrary(u)?; + let times: Range = Arbitrary::arbitrary(u)?; + + Ok(Self { + subspace, + path, + times, + }) + } + + fn size_hint(depth: usize) -> (usize, Option) { + and_all(&[ + AreaSubspace::::size_hint(depth), + Path::::size_hint(depth), + Range::::size_hint(depth), + ]) + } +} + #[cfg(test)] mod tests { diff --git a/data-model/src/grouping/range.rs b/data-model/src/grouping/range.rs index f6de75d..8dd22e6 100644 --- a/data-model/src/grouping/range.rs +++ b/data-model/src/grouping/range.rs @@ -1,6 +1,8 @@ use core::cmp; use core::cmp::Ordering; +use arbitrary::Arbitrary; + #[derive(Debug, PartialEq, Eq)] /// Determines whether a [`Range`] is _closed_ or _open_. pub enum RangeEnd { @@ -20,6 +22,15 @@ impl RangeEnd { } } +impl From<&RangeEnd> for u64 { + fn from(value: &RangeEnd) -> Self { + match value { + RangeEnd::Closed(val) => *val, + RangeEnd::Open => u64::MAX, + } + } +} + impl Ord for RangeEnd { fn cmp(&self, other: &Self) -> Ordering { match self { @@ -41,6 +52,21 @@ impl PartialOrd for RangeEnd { } } +#[cfg(feature = "dev")] +impl<'a> Arbitrary<'a> for RangeEnd { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let is_open: bool = Arbitrary::arbitrary(u)?; + + if !is_open { + let value: u64 = Arbitrary::arbitrary(u)?; + + return Ok(Self::Closed(value)); + } + + Ok(Self::Open) + } +} + /// One-dimensional grouping over a type of value. /// /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#range) @@ -140,6 +166,16 @@ impl PartialOrd for Range { } } +#[cfg(feature = "dev")] +impl<'a> Arbitrary<'a> for Range { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let start: u64 = Arbitrary::arbitrary(u)?; + let end: RangeEnd = Arbitrary::arbitrary(u)?; + + Ok(Self { start, end }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 57ca59a..ab85732 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -203,3 +203,10 @@ path = "fuzz_targets/entry_rel_entry_encoding_random.rs" test = false doc = false bench = false + +[[bin]] +name = "entry_rel_namespace_area_encoding" +path = "fuzz_targets/entry_rel_namespace_area_encoding.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs index 3b8e331..fac29a1 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs @@ -48,6 +48,8 @@ fuzz_target!(|data: ( )| { let (entry_sub, entry_ref, mut consumer) = data; + println!("yay"); + smol::block_on(async { relative_encoding_roundtrip::< Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs new file mode 100644 index 0000000..e8e54b1 --- /dev/null +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs @@ -0,0 +1,69 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; +use willow_data_model::entry::Entry; +use willow_data_model::grouping::area::Area; +use willow_data_model::parameters::PayloadDigest; +use willow_data_model_fuzz::relative_encoding_roundtrip; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encodable for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decodable for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} + +fuzz_target!(|data: ( + Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, + Area<16, 16, 16, IdentityId>, + TestConsumer +)| { + let (entry, area, mut consumer) = data; + + if !area.includes_entry(&entry) { + return; + } + + let namespace = entry.namespace_id.clone(); + + smol::block_on(async { + println!("{:?}", entry); + println!("{:?}", area); + + relative_encoding_roundtrip::< + Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, + (EsNamespaceId, Area<16, 16, 16, IdentityId>), + TestConsumer, + >(entry, (namespace, area), &mut consumer) + .await; + }); +}); From a5fd90d602aa6679276b0e560783704a0b813812 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sat, 20 Jul 2024 18:34:04 +0100 Subject: [PATCH 15/40] Add Entry <> (Namespace, Area) encoding --- data-model/src/encoding/compact_width.rs | 10 +++- data-model/src/encoding/relativity.rs | 40 +++++++------ data-model/src/entry.rs | 6 -- earthstar/src/cinn25519.rs | 29 +++++++++- fuzz/Cargo.toml | 7 +++ fuzz/fuzz_targets/entry_rel_entry_encoding.rs | 2 - .../entry_rel_namespace_area_encoding.rs | 3 - ...ntry_rel_namespace_area_encoding_random.rs | 57 +++++++++++++++++++ fuzz/src/lib.rs | 3 + 9 files changed, 126 insertions(+), 31 deletions(-) create mode 100644 fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index 3df8236..b64af23 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -127,12 +127,20 @@ pub async fn decode_compact_width_be>( compact_width: CompactWidth, producer: &mut Producer, ) -> Result> { - match compact_width { + let decoded = match compact_width { CompactWidth::One => U8BE::decode(producer).await.map(u64::from), CompactWidth::Two => U16BE::decode(producer).await.map(u64::from), CompactWidth::Four => U32BE::decode(producer).await.map(u64::from), CompactWidth::Eight => U64BE::decode(producer).await.map(u64::from), + }?; + + let real_width = CompactWidth::from_u64(decoded); + + if real_width != compact_width { + return Err(DecodeError::InvalidInput); } + + Ok(decoded) } #[cfg(test)] diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 0bc1268..5d45c54 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -246,12 +246,6 @@ where let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; - // Verify that the compact width of the time diff matches the time diff we just decoded. - if CompactWidth::decode_fixed_width_bitmask(header, 4) != CompactWidth::from_u64(time_diff) - { - return Err(DecodeError::InvalidInput); - } - // Add or subtract safely here to avoid overflows caused by malicious or faulty encodings. let timestamp = if add_or_subtract_time_diff { reference @@ -274,13 +268,6 @@ where let payload_length = decode_compact_width_be(compact_width_payload_length, producer).await?; - // Verify that the compact width of the payload length matches the payload length we just decoded. - if CompactWidth::decode_fixed_width_bitmask(header, 6) - != CompactWidth::from_u64(payload_length) - { - return Err(DecodeError::InvalidInput); - } - let payload_digest = PD::decode(producer).await?; Ok(Entry { @@ -315,7 +302,7 @@ where panic!("Tried to encode an entry relative to a namespace it does not belong to") } - if out.includes_entry(self) { + if !out.includes_entry(self) { panic!("Tried to encode an entry relative to an area it is not included by") } @@ -339,7 +326,7 @@ where consumer.consume(header).await?; - if out.subspace != AreaSubspace::Any { + if out.subspace == AreaSubspace::Any { self.subspace_id.encode(consumer).await?; } @@ -356,7 +343,7 @@ impl RelativeDecodable<(N, Area)> for Entry where N: NamespaceId + Decodable, - S: SubspaceId + Decodable, + S: SubspaceId + Decodable + std::fmt::Debug, PD: PayloadDigest + Decodable, { async fn relative_decode( @@ -381,7 +368,10 @@ where } let subspace_id = if is_subspace_encoded { - S::decode(producer).await? + match &out.subspace { + AreaSubspace::Any => S::decode(producer).await?, + AreaSubspace::Id(_) => return Err(DecodeError::InvalidInput), + } } else { match &out.subspace { AreaSubspace::Any => return Err(DecodeError::InvalidInput), @@ -391,6 +381,10 @@ where let path = Path::relative_decode(&out.path, producer).await?; + if !path.is_prefixed_by(&out.path) { + return Err(DecodeError::InvalidInput); + } + let time_diff = decode_compact_width_be(time_diff_compact_width, producer).await?; let payload_length = decode_compact_width_be(payload_length_compact_width, producer).await?; @@ -404,6 +398,18 @@ where } .ok_or(DecodeError::InvalidInput)?; + // Verify that the correct add_or_subtract_time_diff flag was set. + let should_have_added = timestamp.checked_sub(out.times.start) + <= u64::from(&out.times.end).checked_sub(timestamp); + + if add_time_diff_to_start != should_have_added { + return Err(DecodeError::InvalidInput); + } + + if !out.times.includes(×tamp) { + return Err(DecodeError::InvalidInput); + } + Ok(Self { namespace_id: namespace.clone(), subspace_id, diff --git a/data-model/src/entry.rs b/data-model/src/entry.rs index 3aee234..c2803ea 100644 --- a/data-model/src/entry.rs +++ b/data-model/src/entry.rs @@ -135,12 +135,8 @@ where let path: Path = Arbitrary::arbitrary(u)?; - // println!("3"); - let payload_digest: PD = Arbitrary::arbitrary(u)?; - // println!("4"); - Ok(Self { namespace_id, subspace_id, @@ -151,7 +147,6 @@ where }) } - /* fn size_hint(depth: usize) -> (usize, Option) { and_all(&[ N::size_hint(depth), @@ -162,7 +157,6 @@ where u64::size_hint(depth), ]) } - */ } /// An error indicating an [`AuthorisationToken`] does not authorise the writing of this entry. diff --git a/earthstar/src/cinn25519.rs b/earthstar/src/cinn25519.rs index b57933a..2d9472f 100644 --- a/earthstar/src/cinn25519.rs +++ b/earthstar/src/cinn25519.rs @@ -48,9 +48,34 @@ impl<'a, const MIN_LENGTH: usize, const MAX_LENGTH: usize> Arbitrary<'a> for Shortname { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let shortname_string: String = Arbitrary::arbitrary(u)?; + let mut len: usize = Arbitrary::arbitrary(u)?; + len = MIN_LENGTH + (len % (1 + MAX_LENGTH - MIN_LENGTH)); + + let mut s = String::new(); + + if len == 0 { + return Self::new("").map_err(|_| ArbitraryError::IncorrectFormat); + } else { + let mut alpha: u8 = Arbitrary::arbitrary(u)?; + alpha %= 26; // There are 26 lowercase ascii alphabetic chars. + alpha += 0x61; // 0x61 is ASCII 'a'. + s.push(alpha as char); + + for _ in 1..len { + let mut alphanum: u8 = Arbitrary::arbitrary(u)?; + alphanum %= 36; // There are 36 lowercase ascii alphabetic chars or ascii digits. + + if alphanum < 26 { + alphanum += 0x61; + } else { + alphanum = alphanum + 0x30 - 26; // It works, alright? Add the ascii code of '0', but subtract 26, because we transform numbers frmo 26 to 36, not from 0 to 10. (all those ranges with an inclusive start, exclusive end) + } + + s.push(alphanum as char); + } + } - Self::new(&shortname_string).map_err(|_| ArbitraryError::IncorrectFormat) + Self::new(&s).map_err(|_| ArbitraryError::IncorrectFormat) } fn size_hint(_depth: usize) -> (usize, Option) { diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index ab85732..3a71c8a 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -210,3 +210,10 @@ path = "fuzz_targets/entry_rel_namespace_area_encoding.rs" test = false doc = false bench = false + +[[bin]] +name = "entry_rel_namespace_area_encoding_random" +path = "fuzz_targets/entry_rel_namespace_area_encoding_random.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs index fac29a1..3b8e331 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs @@ -48,8 +48,6 @@ fuzz_target!(|data: ( )| { let (entry_sub, entry_ref, mut consumer) = data; - println!("yay"); - smol::block_on(async { relative_encoding_roundtrip::< Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs index e8e54b1..d321953 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs @@ -56,9 +56,6 @@ fuzz_target!(|data: ( let namespace = entry.namespace_id.clone(); smol::block_on(async { - println!("{:?}", entry); - println!("{:?}", area); - relative_encoding_roundtrip::< Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, (EsNamespaceId, Area<16, 16, 16, IdentityId>), diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs new file mode 100644 index 0000000..82705e5 --- /dev/null +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs @@ -0,0 +1,57 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; +use willow_data_model::entry::Entry; +use willow_data_model::grouping::area::Area; +use willow_data_model::parameters::PayloadDigest; +use willow_data_model_fuzz::relative_encoding_random; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encodable for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decodable for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} + +fuzz_target!( + |data: (&[u8], (EsNamespaceId, Area<16, 16, 16, IdentityId>))| { + // fuzzed code goes here + let (random_bytes, namespaced_area) = data; + + smol::block_on(async { + relative_encoding_random::< + (EsNamespaceId, Area<16, 16, 16, IdentityId>), + Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, + >(namespaced_area, random_bytes) + .await; + }); + } +); diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index c8a460a..7942add 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -113,6 +113,9 @@ where // Can we turn it back into the same encoding? let mut consumer = IntoVec::::new(); + //println!("item {:?}", item); + //println!("ref {:?}", reference); + item.relative_encode(&reference, &mut consumer) .await .unwrap(); From 9d684a2c5b766cfc4b153223257838313507a51c Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sat, 20 Jul 2024 18:50:20 +0100 Subject: [PATCH 16/40] =?UTF-8?q?Don=E2=80=99t=20forget=20docs!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data-model/src/encoding/relativity.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 5d45c54..e2f5743 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -288,6 +288,9 @@ where S: SubspaceId + Encodable, PD: PayloadDigest + Encodable, { + /// Encode an [`Entry`] relative to a reference [`NamespaceId`] and [`Area`]. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_entry_in_namespace_area). async fn relative_encode( &self, reference: &(N, Area), @@ -346,6 +349,9 @@ where S: SubspaceId + Decodable + std::fmt::Debug, PD: PayloadDigest + Decodable, { + /// Decode an [`Entry`] relative to a reference [`NamespaceId`] and [`Area`]. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_entry_in_namespace_area). async fn relative_decode( reference: &(N, Area), producer: &mut Producer, From b6e9dde6ada2bdb49d5a4fe69281329463786284 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Sat, 20 Jul 2024 20:47:59 +0200 Subject: [PATCH 17/40] Better Arbitrary impls for Path and Shortname --- data-model/src/path.rs | 82 +++++++++++++++++++++++++++++++------- earthstar/src/cinn25519.rs | 39 +++++++++++++++--- 2 files changed, 102 insertions(+), 19 deletions(-) diff --git a/data-model/src/path.rs b/data-model/src/path.rs index dc7395b..7f0e1e4 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -1,5 +1,5 @@ #[cfg(feature = "dev")] -use arbitrary::{Arbitrary, Error as ArbitraryError, Unstructured}; +use arbitrary::{size_hint::and_all, Arbitrary, Error as ArbitraryError, Unstructured}; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::encoding::{ @@ -38,9 +38,9 @@ impl<'a, const MAX_COMPONENT_LENGTH: usize> Component<'a, MAX_COMPONENT_LENGTH> } /// Create a `Component` from a byte slice, without verifying its length. - /// + /// /// #### Safety - /// + /// /// Supplying a slice of length strictly greater than `MAX_COMPONENT_LENGTH` may trigger undefined behavior, /// either immediately, or on any subsequent function invocation that operates on the resulting [`Component`]. pub unsafe fn new_unchecked(slice: &'a [u8]) -> Self { @@ -98,9 +98,9 @@ impl OwnedComponent { } /// Create an `OwnedComponent` by copying data from a byte slice, without verifying its length. - /// + /// /// #### Safety - /// + /// /// Supplying a slice of length strictly greater than `MAX_COMPONENT_LENGTH` may trigger undefined behavior, /// either immediately, or on any subsequent function invocation that operates on the resulting [`OwnedComponent`]. /// @@ -468,7 +468,7 @@ impl Path { /// Create a new path that consists of the first `length` components. More efficient than creating a new [`Path`] from scratch. /// /// #### Safety - /// + /// /// Undefined behaviour if `length` is greater than `self.get_component_count()`. May manifest directly, or at any later /// function invocation that operates on the resulting [`Path`]. /// @@ -607,7 +607,7 @@ impl Path { /// /// Runs in `O(n)`, where `n` is the total length of the shorter of the two paths. Performs a single allocation to create the return value. pub fn greater_but_not_prefixed(&self) -> Option { - // We iterate through all components in reverse order. For each component, we check whether we can replace it by another cmponent that is strictly greater but not prefixed by the original component. If that is possible, we do replace it with the least such component and drop all later components. If that is impossible, we try again with the previous component. If this impossible for all components, then this functino returns `None`. + // We iterate through all components in reverse order. For each component, we check whether we can replace it by another cmponent that is strictly greater but not prefixed by the original component. If that is possible, we do replace it with the least such component and drop all later components. If that is impossible, we try again with the previous component. If this impossible for all components, then this function returns `None`. for (i, component) in self.components().enumerate().rev() { // If it is possible to append a zero byte to a component, then doing so yields its successor. @@ -833,18 +833,72 @@ impl<'a, const MCL: usize, const MCC: usize, const MPL: usize> Arbitrary<'a> for Path { fn arbitrary(u: &mut Unstructured<'a>) -> Result { - let bytestrings: Vec> = Arbitrary::arbitrary(u)?; - let components: Vec> = bytestrings - .iter() - .filter_map(|bytes| Component::new(bytes)) - .collect(); + let mut total_length_in_bytes: usize = Arbitrary::arbitrary(u)?; + total_length_in_bytes = total_length_in_bytes % (MPL + 1); + + let data: Vec = Arbitrary::arbitrary(u)?; + total_length_in_bytes = core::cmp::min(total_length_in_bytes, data.len()); + + let mut num_components: usize = Arbitrary::arbitrary(u)?; + num_components = num_components % (MCC + 1); + + if num_components == 0 { + total_length_in_bytes = 0; + } + + let buf_capacity = size_of::() * (1 + num_components) + total_length_in_bytes; + let mut buf = BytesMut::with_capacity(buf_capacity); + + // Write the component count of the path as the first usize. + buf.extend_from_slice(&num_components.to_ne_bytes()); - Self::new_from_slice(&components).map_err(|_| ArbitraryError::IncorrectFormat) + let mut length_total_so_far = 0; + for i in 0..num_components { + // The final component must be long enough to result in the total_length_in_bytes. + if i + 1 == num_components { + let final_component_length = total_length_in_bytes - length_total_so_far; + + if final_component_length > MCL { + return Err(ArbitraryError::IncorrectFormat); + } else { + buf.extend_from_slice(&total_length_in_bytes.to_ne_bytes()); + } + } else { + // Any non-final component can take on a random length, ... + let mut component_length: usize = Arbitrary::arbitrary(u)?; + // ... except it must be at most the MCL, and... + component_length = component_length % (MCL + 1); + // ... the total length of all components must not exceed the total path length. + component_length = core::cmp::min( + component_length, + total_length_in_bytes - length_total_so_far, + ); + + length_total_so_far += component_length; + buf.extend_from_slice(&length_total_so_far.to_ne_bytes()); + } + } + + // Finally, add the random path data. + buf.extend_from_slice(&data); + + Ok(Path { + data: HeapEncoding(buf.freeze()), + component_count: num_components, + }) } #[inline] fn size_hint(depth: usize) -> (usize, Option) { - as Arbitrary<'a>>::size_hint(depth) + ( + and_all(&[ + usize::size_hint(depth), + usize::size_hint(depth), + Vec::::size_hint(depth), + ]) + .0, + None, + ) } } diff --git a/earthstar/src/cinn25519.rs b/earthstar/src/cinn25519.rs index b57933a..1b5e4db 100644 --- a/earthstar/src/cinn25519.rs +++ b/earthstar/src/cinn25519.rs @@ -1,4 +1,4 @@ -use arbitrary::{size_hint::and_all, Arbitrary, Error as ArbitraryError}; +use arbitrary::{size_hint::{self, and_all}, Arbitrary, Error as ArbitraryError}; use either::Either; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::encoding::{ @@ -48,13 +48,42 @@ impl<'a, const MIN_LENGTH: usize, const MAX_LENGTH: usize> Arbitrary<'a> for Shortname { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let shortname_string: String = Arbitrary::arbitrary(u)?; + let mut len: usize = Arbitrary::arbitrary(u)?; + len = MIN_LENGTH + (len % (1 + MAX_LENGTH - MIN_LENGTH)); + + let mut s = String::new(); + + if len == 0 { + return Self::new("").map_err(|_| ArbitraryError::IncorrectFormat); + } else { + let mut alpha: u8 = Arbitrary::arbitrary(u)?; + alpha = alpha % 26; // There are 26 lowercase ascii alphabetic chars. + alpha = alpha + 0x61; // 0x61 is ASCII 'a'. + s.push(alpha as char); + + for _ in 1..len { + let mut alphanum: u8 = Arbitrary::arbitrary(u)?; + alphanum = alphanum % 36; // There are 36 lowercase ascii alphabetic chars or ascii digits. + + if alphanum < 26 { + alphanum = alphanum + 0x61; + } else { + alphanum = alphanum + 0x30 - 26; // It works, alright? Add the ascii code of '0', but subtract 26, because we transform numbers frmo 26 to 36, not from 0 to 10. (all those ranges with an inclusive start, exclusive end) + } - Self::new(&shortname_string).map_err(|_| ArbitraryError::IncorrectFormat) + s.push(alphanum as char); + } + } + + Self::new(&s).map_err(|_| ArbitraryError::IncorrectFormat) } - fn size_hint(_depth: usize) -> (usize, Option) { - (MIN_LENGTH, Some(MAX_LENGTH)) + fn size_hint(depth: usize) -> (usize, Option) { + let (u8_min, u8_max) = u8::size_hint(depth); + size_hint::and( + usize::size_hint(depth), + (MIN_LENGTH * u8_min, u8_max.map(|max| max * MAX_LENGTH)), + ) } } From 0d9f384ec6d9ebf7c90f5785c44956d95211fc78 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Sat, 20 Jul 2024 20:49:53 +0200 Subject: [PATCH 18/40] Appease clippy --- data-model/src/path.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data-model/src/path.rs b/data-model/src/path.rs index 7f0e1e4..fe5b387 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -834,13 +834,13 @@ impl<'a, const MCL: usize, const MCC: usize, const MPL: usize> Arbitrary<'a> { fn arbitrary(u: &mut Unstructured<'a>) -> Result { let mut total_length_in_bytes: usize = Arbitrary::arbitrary(u)?; - total_length_in_bytes = total_length_in_bytes % (MPL + 1); + total_length_in_bytes %= MPL + 1; let data: Vec = Arbitrary::arbitrary(u)?; total_length_in_bytes = core::cmp::min(total_length_in_bytes, data.len()); let mut num_components: usize = Arbitrary::arbitrary(u)?; - num_components = num_components % (MCC + 1); + num_components %= MCC + 1; if num_components == 0 { total_length_in_bytes = 0; @@ -867,7 +867,7 @@ impl<'a, const MCL: usize, const MCC: usize, const MPL: usize> Arbitrary<'a> // Any non-final component can take on a random length, ... let mut component_length: usize = Arbitrary::arbitrary(u)?; // ... except it must be at most the MCL, and... - component_length = component_length % (MCL + 1); + component_length %= MCL + 1; // ... the total length of all components must not exceed the total path length. component_length = core::cmp::min( component_length, From 8ad1dd1d0b9df84587e462f54fd3b692564328c9 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Sun, 21 Jul 2024 10:45:56 +0200 Subject: [PATCH 19/40] Fix infinite loop in ufotofu dependencies --- Cargo.lock | 21 ++++++++++++++++----- data-model/Cargo.toml | 2 +- data-model/src/encoding/compact_width.rs | 4 ++-- earthstar/Cargo.toml | 3 ++- fuzz/Cargo.toml | 4 ++-- fuzz/src/lib.rs | 6 +++--- 6 files changed, 26 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index beafb50..8822ae5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,7 +218,7 @@ version = "0.1.0" dependencies = [ "arbitrary", "either", - "ufotofu", + "ufotofu 0.3.0", "willow-data-model", ] @@ -504,9 +504,20 @@ dependencies = [ [[package]] name = "ufotofu" -version = "0.2.0" +version = "0.3.0" +dependencies = [ + "either", + "thiserror-core", + "trait-variant", + "ufotofu_queues", + "wrapper", +] + +[[package]] +name = "ufotofu" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89736749f77fdebcbddcf32574b2f3ba4fd004d39c26abdaef2fcbf93c8e0781" +checksum = "90d1fbdc439b2d48e0725dfe321025033d587530e74a1d7924c32e5e9f2d957a" dependencies = [ "arbitrary", "either", @@ -536,7 +547,7 @@ dependencies = [ "bytes", "either", "smol", - "ufotofu", + "ufotofu 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -547,7 +558,7 @@ dependencies = [ "earthstar", "libfuzzer-sys", "smol", - "ufotofu", + "ufotofu 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "willow-data-model", ] diff --git a/data-model/Cargo.toml b/data-model/Cargo.toml index c5f28e0..ac05cfe 100644 --- a/data-model/Cargo.toml +++ b/data-model/Cargo.toml @@ -10,7 +10,7 @@ dev = ["dep:arbitrary"] [dependencies] either = "1.10.0" arbitrary = { version = "1.0.2", features = ["derive"], optional = true } -ufotofu = "0.2.0" +ufotofu = "0.3.0" bytes = "1.6.0" [dev-dependencies] diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index b64af23..006d8f4 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -146,7 +146,7 @@ pub async fn decode_compact_width_be>( #[cfg(test)] mod tests { use ufotofu::local_nb::consumer::IntoVec; - use ufotofu::local_nb::producer::FromVec; + use ufotofu::local_nb::producer::FromBoxedSlice; use super::*; @@ -230,7 +230,7 @@ mod tests { assert_eq!(decoded_compact_width, compact_width); - let mut producer = FromVec::new(encode_result); + let mut producer = FromBoxedSlice::from_vec(encode_result); let decode_result = decode_compact_width_be(decoded_compact_width, &mut producer) .await diff --git a/earthstar/Cargo.toml b/earthstar/Cargo.toml index e0ed36c..81bfc13 100644 --- a/earthstar/Cargo.toml +++ b/earthstar/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" either = "1.10.0" willow-data-model = { path = "../data-model" } arbitrary = { version = "1.0.2", features = ["derive"]} -ufotofu = "0.2.0" \ No newline at end of file +# ufotofu = "0.2.0" +ufotofu = { path = "../../ufotofu"} \ No newline at end of file diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 3a71c8a..6b22799 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -8,9 +8,9 @@ edition = "2021" cargo-fuzz = true [dependencies] -ufotofu ={ version = "0.2.0", features=["dev"]} +ufotofu = { version = "0.3.0", features=["dev"]} smol = "2.0.0" -arbitrary = { version = "1.0.2", features = ["derive"]} +arbitrary = { version = "1.0.2", features = ["derive"] } libfuzzer-sys = { version = "0.4", features = ["arbitrary-derive"] } diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 7942add..ddda26b 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,7 +1,7 @@ use ufotofu::{ common::consumer::TestConsumer, local_nb::{ - producer::{FromSlice, FromVec}, + producer::{FromSlice, FromBoxedSlice}, BufferedConsumer, BulkConsumer, }, sync::consumer::IntoVec, @@ -31,7 +31,7 @@ where new_vec.extend_from_slice(consumer.as_ref()); // THis should eventually be a testproducer, when we are able to initialise one with some known data. - let mut producer = FromVec::new(new_vec); + let mut producer = FromBoxedSlice::from_vec(new_vec); // Check for correct errors let decoded_item = T::decode(&mut producer).await.unwrap(); @@ -92,7 +92,7 @@ pub async fn relative_encoding_roundtrip( new_vec.extend_from_slice(consumer.as_ref()); // THis should eventually be a testproducer, when we are able to initialise one with some known data. - let mut producer = FromVec::new(new_vec); + let mut producer = FromBoxedSlice::from_vec(new_vec); // Check for correct errors let decoded_item = T::relative_decode(&reference, &mut producer).await.unwrap(); From c5f596501d2dba41bbef97cc5d4fc6f9490006bd Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Sun, 21 Jul 2024 10:49:34 +0200 Subject: [PATCH 20/40] Oops, how about saving before comitting... --- Cargo.lock | 17 +++-------------- earthstar/Cargo.toml | 3 +-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8822ae5..e56fdbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,7 +218,7 @@ version = "0.1.0" dependencies = [ "arbitrary", "either", - "ufotofu 0.3.0", + "ufotofu", "willow-data-model", ] @@ -502,17 +502,6 @@ dependencies = [ "syn", ] -[[package]] -name = "ufotofu" -version = "0.3.0" -dependencies = [ - "either", - "thiserror-core", - "trait-variant", - "ufotofu_queues", - "wrapper", -] - [[package]] name = "ufotofu" version = "0.3.0" @@ -547,7 +536,7 @@ dependencies = [ "bytes", "either", "smol", - "ufotofu 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ufotofu", ] [[package]] @@ -558,7 +547,7 @@ dependencies = [ "earthstar", "libfuzzer-sys", "smol", - "ufotofu 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ufotofu", "willow-data-model", ] diff --git a/earthstar/Cargo.toml b/earthstar/Cargo.toml index 81bfc13..130fa82 100644 --- a/earthstar/Cargo.toml +++ b/earthstar/Cargo.toml @@ -7,5 +7,4 @@ edition = "2021" either = "1.10.0" willow-data-model = { path = "../data-model" } arbitrary = { version = "1.0.2", features = ["derive"]} -# ufotofu = "0.2.0" -ufotofu = { path = "../../ufotofu"} \ No newline at end of file +ufotofu = "0.3.0" \ No newline at end of file From 14926c058fbe8955356bbe8a26af39e941eeff6d Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 21 Jul 2024 13:48:41 +0100 Subject: [PATCH 21/40] Add Entry <> Namespaced Range3d encoding --- data-model/src/encoding/relativity.rs | 239 +++++++++++++++++- data-model/src/grouping/range.rs | 22 +- data-model/src/grouping/range_3d.rs | 17 ++ fuzz/Cargo.toml | 14 + .../entry_rel_namespace_range_encoding.rs | 66 +++++ ...try_rel_namespace_range_encoding_random.rs | 57 +++++ fuzz/src/lib.rs | 7 +- 7 files changed, 413 insertions(+), 9 deletions(-) create mode 100644 fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs create mode 100644 fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index e2f5743..1f6a6ea 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -10,7 +10,11 @@ use crate::{ parameters::{Decodable, Encodable}, }, entry::Entry, - grouping::area::{Area, AreaSubspace}, + grouping::{ + area::{Area, AreaSubspace}, + range::RangeEnd, + range_3d::Range3d, + }, parameters::{NamespaceId, PayloadDigest, SubspaceId}, path::Path, }; @@ -426,3 +430,236 @@ where }) } } + +impl + RelativeEncodable<(N, Range3d)> for Entry +where + N: NamespaceId + Encodable, + S: SubspaceId + Encodable, + PD: PayloadDigest + Encodable, +{ + /// Encode an [`Entry`] relative to a reference [`NamespaceId`] and [`Range3d`]. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_entry_in_namespace_3drange). + async fn relative_encode( + &self, + reference: &(N, Range3d), + consumer: &mut Consumer, + ) -> Result<(), EncodingConsumerError> + where + Consumer: BulkConsumer, + { + let (namespace, out) = reference; + + if &self.namespace_id != namespace { + panic!("Tried to encode an entry relative to a namespace it does not belong to") + } + + if !out.includes_entry(self) { + panic!("Tried to encode an entry relative to a 3d range it is not included by") + } + + let time_diff = core::cmp::min( + self.timestamp.abs_diff(out.times.start), + self.timestamp.abs_diff(u64::from(&out.times.end)), + ); + + let mut header = 0b0000_0000; + + // Encode e.subspace_id? + if self.subspace_id != out.subspaces.start { + header |= 0b1000_0000; + } + + // Encode e.path relative to out.paths.start or to out.paths.end? + let encode_path_relative_to_start = match &out.paths.end { + RangeEnd::Closed(end_path) => { + let start_lcp = self.path.longest_common_prefix(&out.paths.start); + let end_lcp = self.path.longest_common_prefix(end_path); + + start_lcp.get_component_count() >= end_lcp.get_component_count() + } + RangeEnd::Open => true, + }; + + if encode_path_relative_to_start { + header |= 0b0100_0000; + } + + // Add time_diff to out.times.start, or subtract from out.times.end? + let add_time_diff_with_start = time_diff == self.timestamp.abs_diff(out.times.start); + + if add_time_diff_with_start { + header |= 0b0010_0000; + } + + // 2-bit integer n such that 2^n gives compact_width(time_diff) + header |= CompactWidth::from_u64(time_diff).bitmask(4); + + // 2-bit integer n such that 2^n gives compact_width(e.payload_length) + header |= CompactWidth::from_u64(self.payload_length).bitmask(6); + + consumer.consume(header).await?; + + if self.subspace_id != out.subspaces.start { + self.subspace_id.encode(consumer).await?; + } + + // Encode e.path relative to out.paths.start or to out.paths.end? + match &out.paths.end { + RangeEnd::Closed(end_path) => { + if encode_path_relative_to_start { + self.path + .relative_encode(&out.paths.start, consumer) + .await?; + } else { + self.path.relative_encode(end_path, consumer).await?; + } + } + RangeEnd::Open => { + self.path + .relative_encode(&out.paths.start, consumer) + .await?; + } + } + + encode_compact_width_be(time_diff, consumer).await?; + encode_compact_width_be(self.payload_length, consumer).await?; + self.payload_digest.encode(consumer).await?; + + Ok(()) + } +} + +impl + RelativeDecodable<(N, Range3d)> for Entry +where + N: NamespaceId + Decodable, + S: SubspaceId + Decodable + std::fmt::Debug, + PD: PayloadDigest + Decodable, +{ + /// Encode an [`Entry`] relative to a reference [`NamespaceId`] and [`Range3d`]. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_entry_in_namespace_3drange). + async fn relative_decode( + reference: &(N, Range3d), + producer: &mut Producer, + ) -> Result> + where + Producer: BulkProducer, + Self: Sized, + { + let (namespace, out) = reference; + + let header = produce_byte(producer).await?; + + // Decode e.subspace_id? + let is_subspace_encoded = is_bitflagged(header, 0); + + // Decode e.path relative to out.paths.start or to out.paths.end? + let decode_path_relative_to_start = is_bitflagged(header, 1); + + // Add time_diff to out.times.start, or subtract from out.times.end? + let add_time_diff_with_start = is_bitflagged(header, 2); + + if is_bitflagged(header, 3) { + return Err(DecodeError::InvalidInput); + } + + let time_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 4); + let payload_length_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 6); + + let subspace_id = if is_subspace_encoded { + S::decode(producer).await? + } else { + out.subspaces.start.clone() + }; + + // Verify that encoding the subspace was necessary. + if subspace_id == out.subspaces.start && is_subspace_encoded { + return Err(DecodeError::InvalidInput); + } + + // Verify that subspace is included by range + if !out.subspaces.includes(&subspace_id) { + return Err(DecodeError::InvalidInput); + } + + let path = if decode_path_relative_to_start { + Path::relative_decode(&out.paths.start, producer).await? + } else { + match &out.paths.end { + RangeEnd::Closed(end_path) => Path::relative_decode(end_path, producer).await?, + RangeEnd::Open => return Err(DecodeError::InvalidInput), + } + }; + + // Verify that path is included by range + if !out.paths.includes(&path) { + return Err(DecodeError::InvalidInput); + } + + // Verify that the path was encoded relative to the correct bound of the referenc path range. + let should_have_encoded_path_relative_to_start = match &out.paths.end { + RangeEnd::Closed(end_path) => { + let start_lcp = path.longest_common_prefix(&out.paths.start); + let end_lcp = path.longest_common_prefix(end_path); + + start_lcp.get_component_count() >= end_lcp.get_component_count() + } + RangeEnd::Open => true, + }; + + if decode_path_relative_to_start != should_have_encoded_path_relative_to_start { + return Err(DecodeError::InvalidInput); + } + + let time_diff = decode_compact_width_be(time_diff_compact_width, producer).await?; + + let payload_length = + decode_compact_width_be(payload_length_compact_width, producer).await?; + + let payload_digest = PD::decode(producer).await?; + + let timestamp = if add_time_diff_with_start { + out.times.start.checked_add(time_diff) + } else { + match &out.times.end { + RangeEnd::Closed(end_time) => end_time.checked_sub(time_diff), + RangeEnd::Open => u64::from(&out.times.end).checked_sub(time_diff), + } + } + .ok_or(DecodeError::InvalidInput)?; + + // Verify that timestamp is included by range + if !out.times.includes(×tamp) { + return Err(DecodeError::InvalidInput); + } + + // Verify that time_diff is what it should have been + let correct_time_diff = core::cmp::min( + timestamp.abs_diff(out.times.start), + timestamp.abs_diff(u64::from(&out.times.end)), + ); + + if time_diff != correct_time_diff { + return Err(DecodeError::InvalidInput); + } + + // Verify that the combine with start bitflag in the header was correct + let should_have_added_to_start = time_diff == timestamp.abs_diff(out.times.start); + + if should_have_added_to_start != add_time_diff_with_start { + return Err(DecodeError::InvalidInput); + } + + Ok(Self { + namespace_id: namespace.clone(), + subspace_id, + path, + timestamp, + payload_length, + payload_digest, + }) + } +} diff --git a/data-model/src/grouping/range.rs b/data-model/src/grouping/range.rs index 8dd22e6..759179a 100644 --- a/data-model/src/grouping/range.rs +++ b/data-model/src/grouping/range.rs @@ -1,7 +1,7 @@ use core::cmp; use core::cmp::Ordering; -use arbitrary::Arbitrary; +use arbitrary::{Arbitrary, Error as ArbitraryError}; #[derive(Debug, PartialEq, Eq)] /// Determines whether a [`Range`] is _closed_ or _open_. @@ -53,12 +53,15 @@ impl PartialOrd for RangeEnd { } #[cfg(feature = "dev")] -impl<'a> Arbitrary<'a> for RangeEnd { +impl<'a, T> Arbitrary<'a> for RangeEnd +where + T: Arbitrary<'a> + Ord, +{ fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { let is_open: bool = Arbitrary::arbitrary(u)?; if !is_open { - let value: u64 = Arbitrary::arbitrary(u)?; + let value: T = Arbitrary::arbitrary(u)?; return Ok(Self::Closed(value)); } @@ -167,10 +170,17 @@ impl PartialOrd for Range { } #[cfg(feature = "dev")] -impl<'a> Arbitrary<'a> for Range { +impl<'a, T> Arbitrary<'a> for Range +where + T: Arbitrary<'a> + Ord, +{ fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let start: u64 = Arbitrary::arbitrary(u)?; - let end: RangeEnd = Arbitrary::arbitrary(u)?; + let start: T = Arbitrary::arbitrary(u)?; + let end: RangeEnd = Arbitrary::arbitrary(u)?; + + if !end.gt_val(&start) { + return Err(ArbitraryError::IncorrectFormat); + } Ok(Self { start, end }) } diff --git a/data-model/src/grouping/range_3d.rs b/data-model/src/grouping/range_3d.rs index d80098d..94fb447 100644 --- a/data-model/src/grouping/range_3d.rs +++ b/data-model/src/grouping/range_3d.rs @@ -1,3 +1,5 @@ +use arbitrary::Arbitrary; + use crate::{ entry::{Entry, Timestamp}, grouping::{area::Area, range::Range}, @@ -67,6 +69,21 @@ impl } } +#[cfg(feature = "dev")] +impl<'a, const MCL: usize, const MCC: usize, const MPL: usize, S> Arbitrary<'a> + for Range3d +where + S: SubspaceId + Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + Ok(Self { + subspaces: Arbitrary::arbitrary(u)?, + paths: Arbitrary::arbitrary(u)?, + times: Arbitrary::arbitrary(u)?, + }) + } +} + #[cfg(test)] mod tests { diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 3a71c8a..00e3134 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -217,3 +217,17 @@ path = "fuzz_targets/entry_rel_namespace_area_encoding_random.rs" test = false doc = false bench = false + +[[bin]] +name = "entry_rel_namespace_range_encoding" +path = "fuzz_targets/entry_rel_namespace_range_encoding.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "entry_rel_namespace_range_encoding_random" +path = "fuzz_targets/entry_rel_namespace_range_encoding_random.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs new file mode 100644 index 0000000..b57a921 --- /dev/null +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs @@ -0,0 +1,66 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; +use willow_data_model::entry::Entry; +use willow_data_model::grouping::range_3d::Range3d; +use willow_data_model::parameters::PayloadDigest; +use willow_data_model_fuzz::relative_encoding_roundtrip; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encodable for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decodable for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} + +fuzz_target!(|data: ( + Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, + Range3d<16, 16, 16, IdentityId>, + TestConsumer +)| { + let (entry, range_3d, mut consumer) = data; + + if !range_3d.includes_entry(&entry) { + return; + } + + let namespace = entry.namespace_id.clone(); + + smol::block_on(async { + relative_encoding_roundtrip::< + Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, + (EsNamespaceId, Range3d<16, 16, 16, IdentityId>), + TestConsumer, + >(entry, (namespace, range_3d), &mut consumer) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs new file mode 100644 index 0000000..3d8aee3 --- /dev/null +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs @@ -0,0 +1,57 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; +use willow_data_model::encoding::parameters::{Decodable, Encodable}; +use willow_data_model::entry::Entry; +use willow_data_model::grouping::range_3d::Range3d; +use willow_data_model::parameters::PayloadDigest; +use willow_data_model_fuzz::relative_encoding_random; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encodable for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decodable for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} + +fuzz_target!( + |data: (&[u8], (EsNamespaceId, Range3d<16, 16, 16, IdentityId>))| { + // fuzzed code goes here + let (random_bytes, namespaced_range_3d) = data; + + smol::block_on(async { + relative_encoding_random::< + (EsNamespaceId, Range3d<16, 16, 16, IdentityId>), + Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, + >(namespaced_range_3d, random_bytes) + .await; + }); + } +); diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 7942add..fc2734e 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -79,6 +79,9 @@ pub async fn relative_encoding_roundtrip( R: std::fmt::Debug, C: BulkConsumer, { + // println!("item {:?}", subject); + // println!("ref {:?}", reference); + if let Err(_err) = subject.relative_encode(&reference, consumer).await { return; } @@ -113,8 +116,8 @@ where // Can we turn it back into the same encoding? let mut consumer = IntoVec::::new(); - //println!("item {:?}", item); - //println!("ref {:?}", reference); + // println!("item {:?}", item); + // println!("ref {:?}", reference); item.relative_encode(&reference, &mut consumer) .await From 9ceca041b27ce018edb0a2f2c9bbbaddf713d763 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 21 Jul 2024 13:52:37 +0100 Subject: [PATCH 22/40] Move fuzz encoding utils to own module --- fuzz/fuzz_targets/entry_encoding.rs | 2 +- fuzz/fuzz_targets/entry_encoding_random.rs | 2 +- fuzz/fuzz_targets/entry_rel_entry_encoding.rs | 2 +- .../entry_rel_entry_encoding_random.rs | 2 +- .../entry_rel_namespace_area_encoding.rs | 2 +- ...ntry_rel_namespace_area_encoding_random.rs | 2 +- .../entry_rel_namespace_range_encoding.rs | 2 +- ...try_rel_namespace_range_encoding_random.rs | 2 +- fuzz/fuzz_targets/path_encoding.rs | 2 +- fuzz/fuzz_targets/path_encoding_random.rs | 2 +- fuzz/fuzz_targets/path_rel_path_encoding.rs | 2 +- .../path_rel_path_encoding_random.rs | 2 +- fuzz/fuzz_targets/u16be_encoding.rs | 2 +- fuzz/fuzz_targets/u16be_encoding_random.rs | 2 +- fuzz/fuzz_targets/u32be_encoding.rs | 2 +- fuzz/fuzz_targets/u32be_encoding_random.rs | 2 +- fuzz/fuzz_targets/u64be_encoding.rs | 2 +- fuzz/fuzz_targets/u64be_encoding_random.rs | 2 +- fuzz/fuzz_targets/u8be_encoding.rs | 2 +- fuzz/fuzz_targets/u8be_encoding_random.rs | 2 +- fuzz/src/encode.rs | 141 +++++++++++++++++ fuzz/src/lib.rs | 143 +----------------- 22 files changed, 162 insertions(+), 162 deletions(-) create mode 100644 fuzz/src/encode.rs diff --git a/fuzz/fuzz_targets/entry_encoding.rs b/fuzz/fuzz_targets/entry_encoding.rs index 2166c93..bd43cc1 100644 --- a/fuzz/fuzz_targets/entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_encoding.rs @@ -10,7 +10,7 @@ use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::encoding_roundtrip; +use willow_data_model_fuzz::encode::encoding_roundtrip; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/entry_encoding_random.rs b/fuzz/fuzz_targets/entry_encoding_random.rs index 3623546..ac305a6 100644 --- a/fuzz/fuzz_targets/entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_encoding_random.rs @@ -45,7 +45,7 @@ impl Decodable for FakePayloadDigest { impl PayloadDigest for FakePayloadDigest {} use libfuzzer_sys::fuzz_target; -use willow_data_model_fuzz::encoding_random; +use willow_data_model_fuzz::encode::encoding_random; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs index 3b8e331..063dd75 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs @@ -10,7 +10,7 @@ use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::relative_encoding_roundtrip; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs index 84ee8ed..29828a0 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs @@ -9,7 +9,7 @@ use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs index d321953..ac26a10 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs @@ -11,7 +11,7 @@ use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::area::Area; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::relative_encoding_roundtrip; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs index 82705e5..95aaf4b 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs @@ -10,7 +10,7 @@ use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::area::Area; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs index b57a921..e27828a 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs @@ -11,7 +11,7 @@ use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::range_3d::Range3d; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::relative_encoding_roundtrip; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs index 3d8aee3..f58c7d7 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs @@ -10,7 +10,7 @@ use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::range_3d::Range3d; use willow_data_model::parameters::PayloadDigest; -use willow_data_model_fuzz::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct FakePayloadDigest([u8; 32]); diff --git a/fuzz/fuzz_targets/path_encoding.rs b/fuzz/fuzz_targets/path_encoding.rs index 198d4aa..ad4f234 100644 --- a/fuzz/fuzz_targets/path_encoding.rs +++ b/fuzz/fuzz_targets/path_encoding.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::path::Path; -use willow_data_model_fuzz::encoding_roundtrip; +use willow_data_model_fuzz::encode::encoding_roundtrip; const MCL: usize = 111111; const MCC: usize = 111111; diff --git a/fuzz/fuzz_targets/path_encoding_random.rs b/fuzz/fuzz_targets/path_encoding_random.rs index 52b00e7..26d980b 100644 --- a/fuzz/fuzz_targets/path_encoding_random.rs +++ b/fuzz/fuzz_targets/path_encoding_random.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::path::Path; -use willow_data_model_fuzz::encoding_random; +use willow_data_model_fuzz::encode::encoding_random; const MCL: usize = 111111; const MCC: usize = 111111; diff --git a/fuzz/fuzz_targets/path_rel_path_encoding.rs b/fuzz/fuzz_targets/path_rel_path_encoding.rs index 1e79540..7eb4390 100644 --- a/fuzz/fuzz_targets/path_rel_path_encoding.rs +++ b/fuzz/fuzz_targets/path_rel_path_encoding.rs @@ -4,7 +4,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::path::Path; -use willow_data_model_fuzz::relative_encoding_roundtrip; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; const MCL: usize = 16; const MCC: usize = 16; diff --git a/fuzz/fuzz_targets/path_rel_path_encoding_random.rs b/fuzz/fuzz_targets/path_rel_path_encoding_random.rs index 1ed9142..31186a6 100644 --- a/fuzz/fuzz_targets/path_rel_path_encoding_random.rs +++ b/fuzz/fuzz_targets/path_rel_path_encoding_random.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::path::Path; -use willow_data_model_fuzz::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random; const MCL: usize = 16; const MCC: usize = 16; diff --git a/fuzz/fuzz_targets/u16be_encoding.rs b/fuzz/fuzz_targets/u16be_encoding.rs index 48c929d..1413115 100644 --- a/fuzz/fuzz_targets/u16be_encoding.rs +++ b/fuzz/fuzz_targets/u16be_encoding.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::encoding::unsigned_int::U16BE; -use willow_data_model_fuzz::encoding_roundtrip; +use willow_data_model_fuzz::encode::encoding_roundtrip; fuzz_target!(|data: (u16, TestConsumer)| { let (n, mut consumer) = data; diff --git a/fuzz/fuzz_targets/u16be_encoding_random.rs b/fuzz/fuzz_targets/u16be_encoding_random.rs index e8eb070..e664001 100644 --- a/fuzz/fuzz_targets/u16be_encoding_random.rs +++ b/fuzz/fuzz_targets/u16be_encoding_random.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::encoding::unsigned_int::U16BE; -use willow_data_model_fuzz::encoding_random; +use willow_data_model_fuzz::encode::encoding_random; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/fuzz_targets/u32be_encoding.rs b/fuzz/fuzz_targets/u32be_encoding.rs index fb85775..1cda4f7 100644 --- a/fuzz/fuzz_targets/u32be_encoding.rs +++ b/fuzz/fuzz_targets/u32be_encoding.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::encoding::unsigned_int::U32BE; -use willow_data_model_fuzz::encoding_roundtrip; +use willow_data_model_fuzz::encode::encoding_roundtrip; fuzz_target!(|data: (u32, TestConsumer)| { let (n, mut consumer) = data; diff --git a/fuzz/fuzz_targets/u32be_encoding_random.rs b/fuzz/fuzz_targets/u32be_encoding_random.rs index 0dfe412..f0f4924 100644 --- a/fuzz/fuzz_targets/u32be_encoding_random.rs +++ b/fuzz/fuzz_targets/u32be_encoding_random.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::encoding::unsigned_int::U32BE; -use willow_data_model_fuzz::encoding_random; +use willow_data_model_fuzz::encode::encoding_random; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/fuzz_targets/u64be_encoding.rs b/fuzz/fuzz_targets/u64be_encoding.rs index a04e134..2a7781b 100644 --- a/fuzz/fuzz_targets/u64be_encoding.rs +++ b/fuzz/fuzz_targets/u64be_encoding.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::encoding::unsigned_int::U64BE; -use willow_data_model_fuzz::encoding_roundtrip; +use willow_data_model_fuzz::encode::encoding_roundtrip; fuzz_target!(|data: (u64, TestConsumer)| { let (n, mut consumer) = data; diff --git a/fuzz/fuzz_targets/u64be_encoding_random.rs b/fuzz/fuzz_targets/u64be_encoding_random.rs index 1393fe0..7e8acf4 100644 --- a/fuzz/fuzz_targets/u64be_encoding_random.rs +++ b/fuzz/fuzz_targets/u64be_encoding_random.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::encoding::unsigned_int::U64BE; -use willow_data_model_fuzz::encoding_random; +use willow_data_model_fuzz::encode::encoding_random; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/fuzz_targets/u8be_encoding.rs b/fuzz/fuzz_targets/u8be_encoding.rs index 606d46b..d0b9f03 100644 --- a/fuzz/fuzz_targets/u8be_encoding.rs +++ b/fuzz/fuzz_targets/u8be_encoding.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::encoding::unsigned_int::U8BE; -use willow_data_model_fuzz::encoding_roundtrip; +use willow_data_model_fuzz::encode::encoding_roundtrip; fuzz_target!(|data: (u8, TestConsumer)| { let (n, mut consumer) = data; diff --git a/fuzz/fuzz_targets/u8be_encoding_random.rs b/fuzz/fuzz_targets/u8be_encoding_random.rs index 120295a..8082ac1 100644 --- a/fuzz/fuzz_targets/u8be_encoding_random.rs +++ b/fuzz/fuzz_targets/u8be_encoding_random.rs @@ -2,7 +2,7 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::encoding::unsigned_int::U8BE; -use willow_data_model_fuzz::encoding_random; +use willow_data_model_fuzz::encode::encoding_random; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/src/encode.rs b/fuzz/src/encode.rs new file mode 100644 index 0000000..ac47296 --- /dev/null +++ b/fuzz/src/encode.rs @@ -0,0 +1,141 @@ +use ufotofu::{ + common::consumer::TestConsumer, + local_nb::{ + producer::{FromBoxedSlice, FromSlice}, + BufferedConsumer, BulkConsumer, + }, + sync::consumer::IntoVec, +}; + +use willow_data_model::encoding::{ + error::DecodeError, + parameters::{Decodable, Encodable}, + relativity::{RelativeDecodable, RelativeEncodable}, +}; + +pub async fn encoding_roundtrip(item: T, consumer: &mut TestConsumer) +where + T: Encodable + Decodable + std::fmt::Debug + PartialEq + Eq, + C: BulkConsumer, +{ + if let Err(_err) = item.encode(consumer).await { + return; + } + + if let Err(_err) = consumer.flush().await { + return; + } + + let mut new_vec = Vec::new(); + + new_vec.extend_from_slice(consumer.as_ref()); + + // THis should eventually be a testproducer, when we are able to initialise one with some known data. + let mut producer = FromBoxedSlice::from_vec(new_vec); + + // Check for correct errors + let decoded_item = T::decode(&mut producer).await.unwrap(); + + assert_eq!(decoded_item, item); +} + +pub async fn encoding_random(data: &[u8]) +where + T: Encodable + Decodable, +{ + let mut producer = FromSlice::new(data); + + match T::decode(&mut producer).await { + Ok(item) => { + // It decoded to a valid path! Gasp! + // Can we turn it back into the same encoding? + let mut consumer = IntoVec::::new(); + + item.encode(&mut consumer).await.unwrap(); + + let encoded = consumer.as_ref().as_slice(); + + assert_eq!(encoded, &data[0..producer.get_offset()]); + } + Err(err) => match err { + // There was an error. + DecodeError::Producer(_) => panic!("Returned producer error, when whe shouldn't!"), + DecodeError::InvalidInput => { + // GOOD. + } + DecodeError::U64DoesNotFitUsize => { + panic!("Returned u64DoesNotFitUsize error, when we shouldn't!") + } + }, + }; +} + +pub async fn relative_encoding_roundtrip( + subject: T, + reference: R, + consumer: &mut TestConsumer, +) where + T: std::fmt::Debug + PartialEq + Eq + RelativeEncodable + RelativeDecodable, + R: std::fmt::Debug, + C: BulkConsumer, +{ + // println!("item {:?}", subject); + // println!("ref {:?}", reference); + + if let Err(_err) = subject.relative_encode(&reference, consumer).await { + return; + } + + if let Err(_err) = consumer.flush().await { + return; + } + + let mut new_vec = Vec::new(); + + new_vec.extend_from_slice(consumer.as_ref()); + + // THis should eventually be a testproducer, when we are able to initialise one with some known data. + let mut producer = FromBoxedSlice::from_vec(new_vec); + + // Check for correct errors + let decoded_item = T::relative_decode(&reference, &mut producer).await.unwrap(); + + assert_eq!(decoded_item, subject); +} + +pub async fn relative_encoding_random(reference: R, data: &[u8]) +where + T: RelativeEncodable + RelativeDecodable + std::fmt::Debug, + R: std::fmt::Debug, +{ + let mut producer = FromSlice::new(data); + + match T::relative_decode(&reference, &mut producer).await { + Ok(item) => { + // It decoded to a valid item! Gasp! + // Can we turn it back into the same encoding? + let mut consumer = IntoVec::::new(); + + // println!("item {:?}", item); + // println!("ref {:?}", reference); + + item.relative_encode(&reference, &mut consumer) + .await + .unwrap(); + + let encoded = consumer.as_ref().as_slice(); + + assert_eq!(encoded, &data[0..producer.get_offset()]); + } + Err(err) => match err { + // There was an error. + DecodeError::Producer(_) => panic!("Returned producer error, when whe shouldn't!"), + DecodeError::InvalidInput => { + // GOOD. + } + DecodeError::U64DoesNotFitUsize => { + panic!("Returned u64DoesNotFitUsize error, when we shouldn't!") + } + }, + }; +} diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 5bfb5c6..c18eeb8 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,143 +1,2 @@ -use ufotofu::{ - common::consumer::TestConsumer, - local_nb::{ - producer::{FromSlice, FromBoxedSlice}, - BufferedConsumer, BulkConsumer, - }, - sync::consumer::IntoVec, -}; - -use willow_data_model::encoding::{ - error::DecodeError, - parameters::{Decodable, Encodable}, - relativity::{RelativeDecodable, RelativeEncodable}, -}; - -pub async fn encoding_roundtrip(item: T, consumer: &mut TestConsumer) -where - T: Encodable + Decodable + std::fmt::Debug + PartialEq + Eq, - C: BulkConsumer, -{ - if let Err(_err) = item.encode(consumer).await { - return; - } - - if let Err(_err) = consumer.flush().await { - return; - } - - let mut new_vec = Vec::new(); - - new_vec.extend_from_slice(consumer.as_ref()); - - // THis should eventually be a testproducer, when we are able to initialise one with some known data. - let mut producer = FromBoxedSlice::from_vec(new_vec); - - // Check for correct errors - let decoded_item = T::decode(&mut producer).await.unwrap(); - - assert_eq!(decoded_item, item); -} - -pub async fn encoding_random(data: &[u8]) -where - T: Encodable + Decodable, -{ - let mut producer = FromSlice::new(data); - - match T::decode(&mut producer).await { - Ok(item) => { - // It decoded to a valid path! Gasp! - // Can we turn it back into the same encoding? - let mut consumer = IntoVec::::new(); - - item.encode(&mut consumer).await.unwrap(); - - let encoded = consumer.as_ref().as_slice(); - - assert_eq!(encoded, &data[0..producer.get_offset()]); - } - Err(err) => match err { - // There was an error. - DecodeError::Producer(_) => panic!("Returned producer error, when whe shouldn't!"), - DecodeError::InvalidInput => { - // GOOD. - } - DecodeError::U64DoesNotFitUsize => { - panic!("Returned u64DoesNotFitUsize error, when we shouldn't!") - } - }, - }; -} - -pub async fn relative_encoding_roundtrip( - subject: T, - reference: R, - consumer: &mut TestConsumer, -) where - T: std::fmt::Debug + PartialEq + Eq + RelativeEncodable + RelativeDecodable, - R: std::fmt::Debug, - C: BulkConsumer, -{ - // println!("item {:?}", subject); - // println!("ref {:?}", reference); - - if let Err(_err) = subject.relative_encode(&reference, consumer).await { - return; - } - - if let Err(_err) = consumer.flush().await { - return; - } - - let mut new_vec = Vec::new(); - - new_vec.extend_from_slice(consumer.as_ref()); - - // THis should eventually be a testproducer, when we are able to initialise one with some known data. - let mut producer = FromBoxedSlice::from_vec(new_vec); - - // Check for correct errors - let decoded_item = T::relative_decode(&reference, &mut producer).await.unwrap(); - - assert_eq!(decoded_item, subject); -} - -pub async fn relative_encoding_random(reference: R, data: &[u8]) -where - T: RelativeEncodable + RelativeDecodable + std::fmt::Debug, - R: std::fmt::Debug, -{ - let mut producer = FromSlice::new(data); - - match T::relative_decode(&reference, &mut producer).await { - Ok(item) => { - // It decoded to a valid item! Gasp! - // Can we turn it back into the same encoding? - let mut consumer = IntoVec::::new(); - - // println!("item {:?}", item); - // println!("ref {:?}", reference); - - item.relative_encode(&reference, &mut consumer) - .await - .unwrap(); - - let encoded = consumer.as_ref().as_slice(); - - assert_eq!(encoded, &data[0..producer.get_offset()]); - } - Err(err) => match err { - // There was an error. - DecodeError::Producer(_) => panic!("Returned producer error, when whe shouldn't!"), - DecodeError::InvalidInput => { - // GOOD. - } - DecodeError::U64DoesNotFitUsize => { - panic!("Returned u64DoesNotFitUsize error, when we shouldn't!") - } - }, - }; -} - +pub mod encode; pub mod path; From ab61efa1aa9fcbea586bea9e2ec2368e45f26d3a Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 21 Jul 2024 13:57:40 +0100 Subject: [PATCH 23/40] Pull fuzzing FakePayloadDigest out into own module for now --- fuzz/fuzz_targets/entry_encoding.rs | 29 ------------- fuzz/fuzz_targets/entry_encoding_random.rs | 43 +------------------ fuzz/fuzz_targets/entry_rel_entry_encoding.rs | 35 +-------------- .../entry_rel_entry_encoding_random.rs | 35 +-------------- .../entry_rel_namespace_area_encoding.rs | 35 +-------------- ...ntry_rel_namespace_area_encoding_random.rs | 35 +-------------- .../entry_rel_namespace_range_encoding.rs | 35 +-------------- ...try_rel_namespace_range_encoding_random.rs | 35 +-------------- fuzz/src/lib.rs | 1 + fuzz/src/placeholder_params.rs | 38 ++++++++++++++++ 10 files changed, 47 insertions(+), 274 deletions(-) create mode 100644 fuzz/src/placeholder_params.rs diff --git a/fuzz/fuzz_targets/entry_encoding.rs b/fuzz/fuzz_targets/entry_encoding.rs index bd43cc1..f32efa4 100644 --- a/fuzz/fuzz_targets/entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_encoding.rs @@ -12,35 +12,6 @@ use willow_data_model::entry::Entry; use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::encoding_roundtrip; -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} - fuzz_target!(|data: ( Entry<3, 3, 3, EsNamespaceId, IdentityId, FakePayloadDigest>, TestConsumer diff --git a/fuzz/fuzz_targets/entry_encoding_random.rs b/fuzz/fuzz_targets/entry_encoding_random.rs index ac305a6..dc2cedf 100644 --- a/fuzz/fuzz_targets/entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_encoding_random.rs @@ -1,51 +1,12 @@ #![no_main] -use arbitrary::Arbitrary; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; - use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use willow_data_model::{ - encoding::{ - error::{DecodeError, EncodingConsumerError}, - parameters::{Decodable, Encodable}, - }, - entry::Entry, - parameters::PayloadDigest, -}; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model::entry::Entry; use libfuzzer_sys::fuzz_target; -use willow_data_model_fuzz::encode::encoding_random; +use willow_data_model_fuzz::{encode::encoding_random, placeholder_params::FakePayloadDigest}; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs index 063dd75..e52377e 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding.rs @@ -2,44 +2,11 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::relative_encoding_roundtrip; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!(|data: ( Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs index 29828a0..a110ef7 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs @@ -2,43 +2,10 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::relative_encoding_random; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!(|data: ( &[u8], diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs index ac26a10..3b2439a 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs @@ -2,45 +2,12 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::area::Area; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::relative_encoding_roundtrip; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!(|data: ( Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs index 95aaf4b..6272a8b 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs @@ -2,44 +2,11 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::area::Area; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::relative_encoding_random; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!( |data: (&[u8], (EsNamespaceId, Area<16, 16, 16, IdentityId>))| { diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs index e27828a..c5a9053 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs @@ -2,45 +2,12 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::range_3d::Range3d; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::relative_encoding_roundtrip; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!(|data: ( Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs index f58c7d7..cc0e69b 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs @@ -2,44 +2,11 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; use willow_data_model::grouping::range_3d::Range3d; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::relative_encoding_random; - -#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct FakePayloadDigest([u8; 32]); - -impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> - where - C: BulkConsumer, - { - consumer.bulk_consume_full_slice(&self.0).await?; - - Ok(()) - } -} - -impl Decodable for FakePayloadDigest { - async fn decode

(producer: &mut P) -> Result> - where - P: BulkProducer, - { - let mut slice = [0u8; 32]; - - producer.bulk_overwrite_full_slice(&mut slice).await?; - - Ok(FakePayloadDigest(slice)) - } -} - -impl PayloadDigest for FakePayloadDigest {} +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!( |data: (&[u8], (EsNamespaceId, Range3d<16, 16, 16, IdentityId>))| { diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index c18eeb8..a3877e7 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,2 +1,3 @@ pub mod encode; pub mod path; +pub mod placeholder_params; diff --git a/fuzz/src/placeholder_params.rs b/fuzz/src/placeholder_params.rs new file mode 100644 index 0000000..3725907 --- /dev/null +++ b/fuzz/src/placeholder_params.rs @@ -0,0 +1,38 @@ +use arbitrary::Arbitrary; +use ufotofu::local_nb::{BulkConsumer, BulkProducer}; +use willow_data_model::{ + encoding::{ + error::{DecodeError, EncodingConsumerError}, + parameters::{Decodable, Encodable}, + }, + parameters::PayloadDigest, +}; + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct FakePayloadDigest([u8; 32]); + +impl Encodable for FakePayloadDigest { + async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + where + C: BulkConsumer, + { + consumer.bulk_consume_full_slice(&self.0).await?; + + Ok(()) + } +} + +impl Decodable for FakePayloadDigest { + async fn decode

(producer: &mut P) -> Result> + where + P: BulkProducer, + { + let mut slice = [0u8; 32]; + + producer.bulk_overwrite_full_slice(&mut slice).await?; + + Ok(FakePayloadDigest(slice)) + } +} + +impl PayloadDigest for FakePayloadDigest {} From db580c23ad774a09d74f67e9224c5f77a50c644f Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 21 Jul 2024 14:43:10 +0100 Subject: [PATCH 24/40] Remove EncodeConsumerError --- data-model/src/encoding/compact_width.rs | 7 ++-- data-model/src/encoding/error.rs | 33 +------------------ data-model/src/encoding/max_power.rs | 7 ++-- data-model/src/encoding/parameters.rs | 4 +-- data-model/src/encoding/relativity.rs | 12 +++---- data-model/src/encoding/unsigned_int.rs | 42 ++++++++++++------------ data-model/src/entry.rs | 4 +-- data-model/src/path.rs | 9 +++-- earthstar/src/cinn25519.rs | 22 ++++++++----- earthstar/src/identity_id.rs | 4 +-- earthstar/src/namespace_id.rs | 4 +-- fuzz/fuzz_targets/entry_encoding.rs | 6 +--- fuzz/src/placeholder_params.rs | 9 +++-- 13 files changed, 71 insertions(+), 92 deletions(-) diff --git a/data-model/src/encoding/compact_width.rs b/data-model/src/encoding/compact_width.rs index 006d8f4..e988d48 100644 --- a/data-model/src/encoding/compact_width.rs +++ b/data-model/src/encoding/compact_width.rs @@ -1,4 +1,4 @@ -use crate::encoding::error::{DecodeError, EncodingConsumerError}; +use crate::encoding::error::DecodeError; use crate::encoding::parameters::Decodable; use crate::encoding::unsigned_int::{U16BE, U32BE, U64BE, U8BE}; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; @@ -112,12 +112,13 @@ impl CompactWidth { pub async fn encode_compact_width_be>( value: u64, consumer: &mut Consumer, -) -> Result<(), EncodingConsumerError> { +) -> Result<(), Consumer::Error> { let width = CompactWidth::from_u64(value).width(); consumer .bulk_consume_full_slice(&value.to_be_bytes()[8 - width..]) - .await?; + .await + .map_err(|f| f.reason)?; Ok(()) } diff --git a/data-model/src/encoding/error.rs b/data-model/src/encoding/error.rs index 0d49e57..4e7deed 100644 --- a/data-model/src/encoding/error.rs +++ b/data-model/src/encoding/error.rs @@ -1,38 +1,7 @@ use core::error::Error; use core::{fmt::Display, fmt::Formatter, num::TryFromIntError}; use either::Either; -use ufotofu::common::errors::{ConsumeFullSliceError, OverwriteFullSliceError}; - -/// Returned when a encoding fails to be consumed by a [`ufotofu::local_nb::Consumer`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EncodingConsumerError(E); - -impl From> for EncodingConsumerError { - fn from(err: ConsumeFullSliceError) -> Self { - EncodingConsumerError(err.reason) - } -} - -impl From for EncodingConsumerError { - fn from(value: E) -> Self { - Self(value) - } -} - -impl Error for EncodingConsumerError -where - E: 'static + Error, -{ - fn source(&self) -> Option<&(dyn Error + 'static)> { - Some(&self.0) - } -} - -impl Display for EncodingConsumerError { - fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { - write!(f, "The consumer returned an error after trying to consume",) - } -} +use ufotofu::common::errors::OverwriteFullSliceError; /// Everything that can go wrong when decoding a value. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/data-model/src/encoding/max_power.rs b/data-model/src/encoding/max_power.rs index 286c14a..0092724 100644 --- a/data-model/src/encoding/max_power.rs +++ b/data-model/src/encoding/max_power.rs @@ -1,7 +1,7 @@ use core::mem::size_of; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use crate::encoding::error::{DecodeError, EncodingConsumerError}; +use crate::encoding::error::DecodeError; /// Return the least natural number such that 256^`n` is greater than or equal to `n`. /// @@ -31,7 +31,7 @@ pub async fn encode_max_power( value: usize, max_size: usize, consumer: &mut C, -) -> Result<(), EncodingConsumerError> +) -> Result<(), C::Error> where C: BulkConsumer, { @@ -44,7 +44,8 @@ where consumer .bulk_consume_full_slice(&value_encoded_raw[size_of::() - (power as usize)..]) - .await?; + .await + .map_err(|f| f.reason)?; Ok(()) } diff --git a/data-model/src/encoding/parameters.rs b/data-model/src/encoding/parameters.rs index 9d93e1f..cf3d458 100644 --- a/data-model/src/encoding/parameters.rs +++ b/data-model/src/encoding/parameters.rs @@ -1,6 +1,6 @@ use std::future::Future; -use crate::encoding::error::{DecodeError, EncodingConsumerError}; +use crate::encoding::error::DecodeError; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; /// A type that can be encoded to a bytestring, ensuring that any value of `Self` maps to exactly one bytestring. @@ -13,7 +13,7 @@ pub trait Encodable { fn encode( &self, consumer: &mut Consumer, - ) -> impl Future>> + ) -> impl Future> where Consumer: BulkConsumer; } diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 1f6a6ea..095d364 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -5,7 +5,7 @@ use crate::{ encoding::{ bytes::{is_bitflagged, produce_byte}, compact_width::{decode_compact_width_be, encode_compact_width_be, CompactWidth}, - error::{DecodeError, EncodingConsumerError}, + error::DecodeError, max_power::{decode_max_power, encode_max_power}, parameters::{Decodable, Encodable}, }, @@ -26,7 +26,7 @@ pub trait RelativeEncodable { &self, reference: &R, consumer: &mut Consumer, - ) -> impl Future>> + ) -> impl Future> where Consumer: BulkConsumer; } @@ -55,7 +55,7 @@ impl RelativeEncodable, consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + ) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { @@ -145,7 +145,7 @@ where &self, reference: &Entry, consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + ) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { @@ -299,7 +299,7 @@ where &self, reference: &(N, Area), consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + ) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { @@ -445,7 +445,7 @@ where &self, reference: &(N, Range3d), consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + ) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { diff --git a/data-model/src/encoding/unsigned_int.rs b/data-model/src/encoding/unsigned_int.rs index 7be4c30..edbd2ab 100644 --- a/data-model/src/encoding/unsigned_int.rs +++ b/data-model/src/encoding/unsigned_int.rs @@ -1,7 +1,7 @@ use core::mem::size_of; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use crate::encoding::error::{DecodeError, EncodingConsumerError}; +use crate::encoding::error::DecodeError; use crate::encoding::parameters::{Decodable, Encodable}; /// A `u8` wrapper that implements [`Encoding`] and [`Decoding`] by encoding as a big-endian fixed-width integer. @@ -9,15 +9,15 @@ use crate::encoding::parameters::{Decodable, Encodable}; pub struct U8BE(u8); impl Encodable for U8BE { - async fn encode( - &self, - consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut Consumer) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { let byte = self.0.to_be_bytes(); - consumer.bulk_consume_full_slice(&byte).await?; + consumer + .bulk_consume_full_slice(&byte) + .await + .map_err(|f| f.reason)?; Ok(()) } } @@ -51,15 +51,15 @@ impl From for u64 { pub struct U16BE(u16); impl Encodable for U16BE { - async fn encode( - &self, - consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut Consumer) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { let bytes = self.0.to_be_bytes(); - consumer.bulk_consume_full_slice(&bytes).await?; + consumer + .bulk_consume_full_slice(&bytes) + .await + .map_err(|f| f.reason)?; Ok(()) } } @@ -93,15 +93,15 @@ impl From for u64 { pub struct U32BE(u32); impl Encodable for U32BE { - async fn encode( - &self, - consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut Consumer) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { let bytes = self.0.to_be_bytes(); - consumer.bulk_consume_full_slice(&bytes).await?; + consumer + .bulk_consume_full_slice(&bytes) + .await + .map_err(|f| f.reason)?; Ok(()) } } @@ -135,15 +135,15 @@ impl From for u64 { pub struct U64BE(u64); impl Encodable for U64BE { - async fn encode( - &self, - consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut Consumer) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { let bytes = self.0.to_be_bytes(); - consumer.bulk_consume_full_slice(&bytes).await?; + consumer + .bulk_consume_full_slice(&bytes) + .await + .map_err(|f| f.reason)?; Ok(()) } } diff --git a/data-model/src/entry.rs b/data-model/src/entry.rs index c2803ea..a58b3a7 100644 --- a/data-model/src/entry.rs +++ b/data-model/src/entry.rs @@ -6,7 +6,7 @@ use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ encoding::{ - error::{DecodeError, EncodingConsumerError}, + error::DecodeError, parameters::{Decodable, Encodable}, unsigned_int::U64BE, }, @@ -74,7 +74,7 @@ where S: SubspaceId + Encodable, PD: PayloadDigest + Encodable, { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError<::Error>> + async fn encode(&self, consumer: &mut C) -> Result<(), ::Error> where C: BulkConsumer, { diff --git a/data-model/src/path.rs b/data-model/src/path.rs index fe5b387..42d17e1 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -3,7 +3,7 @@ use arbitrary::{size_hint::and_all, Arbitrary, Error as ArbitraryError, Unstruct use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::encoding::{ - error::{DecodeError, EncodingConsumerError}, + error::DecodeError, max_power::{decode_max_power, encode_max_power}, parameters::{Decodable, Encodable}, }; @@ -784,7 +784,7 @@ impl AsRef<[u8]> for HeapEncoding Encodable for Path { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut C) -> Result<(), C::Error> where C: BulkConsumer, { @@ -793,7 +793,10 @@ impl Encodable for Path Cinn25519PublicKey Encodable for Cinn25519PublicKey { - async fn encode( - &self, - consumer: &mut Consumer, - ) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut Consumer) -> Result<(), Consumer::Error> where Consumer: BulkConsumer, { @@ -111,13 +111,19 @@ impl Encodable vec.extend_from_slice(&self.shortname.0); - consumer.bulk_consume_full_slice(&self.shortname.0).await?; + consumer + .bulk_consume_full_slice(&self.shortname.0) + .await + .map_err(|f| f.reason)?; if MIN_LENGTH < MAX_LENGTH { consumer.consume(0x0).await?; } - consumer.bulk_consume_full_slice(&self.underlying).await?; + consumer + .bulk_consume_full_slice(&self.underlying) + .await + .map_err(|f| f.reason)?; Ok(()) } diff --git a/earthstar/src/identity_id.rs b/earthstar/src/identity_id.rs index d90d7ca..7711eba 100644 --- a/earthstar/src/identity_id.rs +++ b/earthstar/src/identity_id.rs @@ -2,7 +2,7 @@ use arbitrary::Arbitrary; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::{ encoding::{ - error::{DecodeError, EncodingConsumerError}, + error::DecodeError, parameters::{Decodable, Encodable}, }, parameters::SubspaceId, @@ -23,7 +23,7 @@ impl Default for IdentityIdentifier { } impl Encodable for IdentityIdentifier { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut C) -> Result<(), C::Error> where C: BulkConsumer, { diff --git a/earthstar/src/namespace_id.rs b/earthstar/src/namespace_id.rs index b4e5da0..6a5aa61 100644 --- a/earthstar/src/namespace_id.rs +++ b/earthstar/src/namespace_id.rs @@ -2,7 +2,7 @@ use arbitrary::Arbitrary; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::{ encoding::{ - error::{DecodeError, EncodingConsumerError}, + error::DecodeError, parameters::{Decodable, Encodable}, }, parameters::NamespaceId, @@ -23,7 +23,7 @@ impl Default for NamespaceIdentifier { } impl Encodable for NamespaceIdentifier { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut C) -> Result<(), C::Error> where C: BulkConsumer, { diff --git a/fuzz/fuzz_targets/entry_encoding.rs b/fuzz/fuzz_targets/entry_encoding.rs index f32efa4..4043c7a 100644 --- a/fuzz/fuzz_targets/entry_encoding.rs +++ b/fuzz/fuzz_targets/entry_encoding.rs @@ -2,15 +2,11 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; -use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use ufotofu::local_nb::consumer::TestConsumer; -use ufotofu::local_nb::{BulkConsumer, BulkProducer}; -use willow_data_model::encoding::error::{DecodeError, EncodingConsumerError}; -use willow_data_model::encoding::parameters::{Decodable, Encodable}; use willow_data_model::entry::Entry; -use willow_data_model::parameters::PayloadDigest; use willow_data_model_fuzz::encode::encoding_roundtrip; +use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!(|data: ( Entry<3, 3, 3, EsNamespaceId, IdentityId, FakePayloadDigest>, diff --git a/fuzz/src/placeholder_params.rs b/fuzz/src/placeholder_params.rs index 3725907..979e369 100644 --- a/fuzz/src/placeholder_params.rs +++ b/fuzz/src/placeholder_params.rs @@ -2,7 +2,7 @@ use arbitrary::Arbitrary; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use willow_data_model::{ encoding::{ - error::{DecodeError, EncodingConsumerError}, + error::DecodeError, parameters::{Decodable, Encodable}, }, parameters::PayloadDigest, @@ -12,11 +12,14 @@ use willow_data_model::{ pub struct FakePayloadDigest([u8; 32]); impl Encodable for FakePayloadDigest { - async fn encode(&self, consumer: &mut C) -> Result<(), EncodingConsumerError> + async fn encode(&self, consumer: &mut C) -> Result<(), C::Error> where C: BulkConsumer, { - consumer.bulk_consume_full_slice(&self.0).await?; + consumer + .bulk_consume_full_slice(&self.0) + .await + .map_err(|f| f.reason)?; Ok(()) } From da6f1bc2333dae68a1bb716149ce803f60d19df5 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Sun, 21 Jul 2024 16:50:11 +0100 Subject: [PATCH 25/40] Area <> Area relative encoder --- data-model/src/encoding/relativity.rs | 251 +++++++++++++++++- data-model/src/grouping/area.rs | 20 +- fuzz/Cargo.toml | 14 + fuzz/fuzz_targets/area_rel_area_encoding.rs | 28 ++ .../area_rel_area_encoding_random.rs | 19 ++ 5 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 fuzz/fuzz_targets/area_rel_area_encoding.rs create mode 100644 fuzz/fuzz_targets/area_rel_area_encoding_random.rs diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 095d364..494b0b3 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -12,7 +12,7 @@ use crate::{ entry::Entry, grouping::{ area::{Area, AreaSubspace}, - range::RangeEnd, + range::{Range, RangeEnd}, range_3d::Range3d, }, parameters::{NamespaceId, PayloadDigest, SubspaceId}, @@ -538,7 +538,7 @@ where S: SubspaceId + Decodable + std::fmt::Debug, PD: PayloadDigest + Decodable, { - /// Encode an [`Entry`] relative to a reference [`NamespaceId`] and [`Range3d`]. + /// Decode an [`Entry`] relative to a reference [`NamespaceId`] and [`Range3d`]. /// /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_entry_in_namespace_3drange). async fn relative_decode( @@ -663,3 +663,250 @@ where }) } } + +impl + RelativeEncodable> for Area +where + S: SubspaceId + Encodable, +{ + /// Encode an [`Area`] relative to another [`Area`] which [includes](https://willowprotocol.org/specs/grouping-entries/index.html#area_include_area) it. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_area_in_area). + async fn relative_encode( + &self, + out: &Area, + consumer: &mut Consumer, + ) -> Result<(), Consumer::Error> + where + Consumer: BulkConsumer, + { + if !out.includes_area(self) { + panic!("Tried to encode an area relative to a area it is not included by") + } + + let start_diff = core::cmp::min( + self.times.start - out.times.start, + u64::from(&out.times.end) - self.times.start, + ); + + let end_diff = core::cmp::min( + u64::from(&self.times.end) - out.times.start, + u64::from(&out.times.end) - u64::from(&self.times.end), + ); + + let mut header = 0; + + if self.subspace != out.subspace { + header |= 0b1000_0000; + } + + if self.times.end == RangeEnd::Open { + header |= 0b0100_0000; + } + + if start_diff == self.times.start - out.times.start { + header |= 0b0010_0000; + } + + if self.times.end != RangeEnd::Open + && end_diff == u64::from(&self.times.end) - out.times.start + { + header |= 0b0001_0000; + } + + header |= CompactWidth::from_u64(start_diff).bitmask(4); + header |= CompactWidth::from_u64(end_diff).bitmask(6); + + // println!("enc header: {:08b}", header); + + consumer.consume(header).await?; + + match (&self.subspace, &out.subspace) { + (AreaSubspace::Any, AreaSubspace::Any) => {} // Same subspace + (AreaSubspace::Id(_), AreaSubspace::Id(_)) => {} // Same subspace + (AreaSubspace::Id(subspace), AreaSubspace::Any) => { + subspace.encode(consumer).await?; + } + (AreaSubspace::Any, AreaSubspace::Id(_)) => { + unreachable!( + "We should have already rejected an area not included by another area!" + ) + } + } + + self.path.relative_encode(&out.path, consumer).await?; + + encode_compact_width_be(start_diff, consumer).await?; + + if self.times.end != RangeEnd::Open { + encode_compact_width_be(end_diff, consumer).await?; + } + + Ok(()) + } +} + +impl + RelativeDecodable> for Area +where + S: SubspaceId + Decodable, +{ + /// Decode an [`Area`] relative to another [`Area`] which [includes](https://willowprotocol.org/specs/grouping-entries/index.html#area_include_area) it. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_area_in_area). + async fn relative_decode( + out: &Area, + producer: &mut Producer, + ) -> Result> + where + Producer: BulkProducer, + Self: Sized, + { + let header = produce_byte(producer).await?; + + // println!("dec header: {:08b}", header); + + // Decode subspace? + let is_subspace_encoded = is_bitflagged(header, 0); + + // Decode end value of times? + let is_times_end_open = is_bitflagged(header, 1); + + // Add start_diff to out.times.start, or subtract from out.times.end? + let add_start_diff = is_bitflagged(header, 2); + + // Add end_diff to out.times.start, or subtract from out.times.end? + let add_end_diff = is_bitflagged(header, 3); + + // Verify that we don't add_end_diff when open... + + if add_end_diff && is_times_end_open { + return Err(DecodeError::InvalidInput); + } + + let start_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 4); + let end_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 6); + + // Verify the last two bits are zero if is_times_end_open + if is_times_end_open && (end_diff_compact_width != CompactWidth::One) { + return Err(DecodeError::InvalidInput); + } + + let subspace = if is_subspace_encoded { + let id = S::decode(producer).await?; + let sub = AreaSubspace::Id(id); + + // Verify that subspace wasn't needlessly encoded + if sub == out.subspace { + return Err(DecodeError::InvalidInput); + } + + sub + } else { + out.subspace.clone() + }; + + // Verify that the decoded subspace is included by the reference subspace + match (&out.subspace, &subspace) { + (AreaSubspace::Any, AreaSubspace::Any) => {} + (AreaSubspace::Any, AreaSubspace::Id(_)) => {} + (AreaSubspace::Id(_), AreaSubspace::Any) => { + return Err(DecodeError::InvalidInput); + } + (AreaSubspace::Id(a), AreaSubspace::Id(b)) => { + if a != b { + return Err(DecodeError::InvalidInput); + } + } + } + + let path = Path::relative_decode(&out.path, producer).await?; + + // Verify the decoded path is prefixed by the reference path + if !path.is_prefixed_by(&out.path) { + return Err(DecodeError::InvalidInput); + } + + let start_diff = decode_compact_width_be(start_diff_compact_width, producer).await?; + + let start = if add_start_diff { + out.times.start.checked_add(start_diff) + } else { + u64::from(&out.times.end).checked_sub(start_diff) + } + .ok_or(DecodeError::InvalidInput)?; + + // Verify they sent correct start diff + let expected_start_diff = core::cmp::min( + start.checked_sub(out.times.start), + u64::from(&out.times.end).checked_sub(start), + ) + .ok_or(DecodeError::InvalidInput)?; + + if expected_start_diff != start_diff { + return Err(DecodeError::InvalidInput); + } + + // Verify that bit 2 of the header was set correctly + let should_add_start_diff = start_diff + == start + .checked_sub(out.times.start) + .ok_or(DecodeError::InvalidInput)?; + + if add_start_diff != should_add_start_diff { + return Err(DecodeError::InvalidInput); + } + + let end = if is_times_end_open { + if add_end_diff { + return Err(DecodeError::InvalidInput); + } + + RangeEnd::Open + } else { + let end_diff = decode_compact_width_be(end_diff_compact_width, producer).await?; + + let end = if add_end_diff { + out.times.start.checked_add(end_diff) + } else { + u64::from(&out.times.end).checked_sub(end_diff) + } + .ok_or(DecodeError::InvalidInput)?; + + // Verify they sent correct end diff + let expected_end_diff = core::cmp::min( + end.checked_sub(out.times.start), + u64::from(&out.times.end).checked_sub(end), + ) + .ok_or(DecodeError::InvalidInput)?; + + if end_diff != expected_end_diff { + return Err(DecodeError::InvalidInput); + } + + let should_add_end_diff = end_diff + == end + .checked_sub(out.times.start) + .ok_or(DecodeError::InvalidInput)?; + + if add_end_diff != should_add_end_diff { + return Err(DecodeError::InvalidInput); + } + + RangeEnd::Closed(end) + }; + + let times = Range { start, end }; + + // Verify the decoded time range is included by the reference time range + if !out.times.includes_range(×) { + return Err(DecodeError::InvalidInput); + } + + Ok(Self { + subspace, + path, + times, + }) + } +} diff --git a/data-model/src/grouping/area.rs b/data-model/src/grouping/area.rs index e82c8d9..10a274f 100644 --- a/data-model/src/grouping/area.rs +++ b/data-model/src/grouping/area.rs @@ -9,7 +9,7 @@ use crate::{ use super::range::Range; /// The possible values of an [`Area`]'s `subspace`. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] pub enum AreaSubspace { /// A value that signals that an [`Area`] includes Entries with arbitrary subspace_ids. Any, @@ -99,6 +99,24 @@ impl Area bool { + match (&self.subspace, &area.subspace) { + (AreaSubspace::Any, AreaSubspace::Any) => { + self.path.is_prefix_of(&area.path) && self.times.includes_range(&area.times) + } + (AreaSubspace::Any, AreaSubspace::Id(_)) => { + self.path.is_prefix_of(&area.path) && self.times.includes_range(&area.times) + } + (AreaSubspace::Id(_), AreaSubspace::Any) => false, + (AreaSubspace::Id(subspace_a), AreaSubspace::Id(subspace_b)) => { + subspace_a == subspace_b + && self.path.is_prefix_of(&area.path) + && self.times.includes_range(&area.times) + } + } + } + /// Return the intersection of this [`Area`] with another. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#area_intersection). pub fn intersection(&self, other: &Area) -> Option { diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 25fd224..e3b8eba 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -231,3 +231,17 @@ path = "fuzz_targets/entry_rel_namespace_range_encoding_random.rs" test = false doc = false bench = false + +[[bin]] +name = "area_rel_area_encoding" +path = "fuzz_targets/area_rel_area_encoding.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "area_rel_area_encoding_random" +path = "fuzz_targets/area_rel_area_encoding_random.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/area_rel_area_encoding.rs b/fuzz/fuzz_targets/area_rel_area_encoding.rs new file mode 100644 index 0000000..0718710 --- /dev/null +++ b/fuzz/fuzz_targets/area_rel_area_encoding.rs @@ -0,0 +1,28 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use willow_data_model::grouping::area::Area; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; + +fuzz_target!(|data: ( + Area<16, 16, 16, IdentityId>, + Area<16, 16, 16, IdentityId>, + TestConsumer +)| { + let (a, out, mut consumer) = data; + + if !out.includes_area(&a) { + return; + } + + smol::block_on(async { + relative_encoding_roundtrip::< + Area<16, 16, 16, IdentityId>, + Area<16, 16, 16, IdentityId>, + TestConsumer, + >(a, out, &mut consumer) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/area_rel_area_encoding_random.rs b/fuzz/fuzz_targets/area_rel_area_encoding_random.rs new file mode 100644 index 0000000..a2f9917 --- /dev/null +++ b/fuzz/fuzz_targets/area_rel_area_encoding_random.rs @@ -0,0 +1,19 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use libfuzzer_sys::fuzz_target; +use willow_data_model::grouping::area::Area; +use willow_data_model_fuzz::encode::relative_encoding_random; + +fuzz_target!(|data: (&[u8], Area<16, 16, 16, IdentityId>)| { + // fuzzed code goes here + let (random_bytes, area) = data; + + smol::block_on(async { + relative_encoding_random::, Area<16, 16, 16, IdentityId>>( + area, + random_bytes, + ) + .await; + }); +}); From 35efbbeb64395dd10ef0ad0d03b407b3e47209d8 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Mon, 22 Jul 2024 13:49:23 +0200 Subject: [PATCH 26/40] Implement zero-allocation relative encoding --- data-model/src/encoding/relativity.rs | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 494b0b3..59bdc59 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -60,27 +60,22 @@ impl RelativeEncodable, { let lcp = self.longest_common_prefix(reference); - encode_max_power(lcp.get_component_count(), MCC, consumer).await?; + let lcp_component_count = lcp.get_component_count(); + encode_max_power(lcp_component_count, MCC, consumer).await?; + + let suffix_component_count = self.get_component_count() - lcp_component_count; + encode_max_power(suffix_component_count, MCC, consumer).await?; - if lcp.get_component_count() > 0 { - let suffix_components = self.suffix_components(lcp.get_component_count()); - // TODO: A more performant version of this. + for component in self.suffix_components(lcp_component_count) { + encode_max_power(component.len(), MCL, consumer).await?; - let mut suffix = Path::::new_empty(); - - for component in suffix_components { - // We can unwrap here because this suffix is a subset of a valid path. - suffix = suffix.append(component).unwrap(); - } - - suffix.encode(consumer).await?; - - return Ok(()); + consumer + .bulk_consume_full_slice(component.as_ref()) + .await + .map_err(|f| f.reason)?; } - self.encode(consumer).await?; - Ok(()) } } From 60376dec56fd309c512aa9c8d30cd6eda3032d15 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 22 Jul 2024 16:39:28 +0100 Subject: [PATCH 27/40] Range3d <> Range3d relative encoding --- data-model/src/encoding/relativity.rs | 344 +++++++++++++++++- data-model/src/grouping/range.rs | 66 ++++ fuzz/Cargo.toml | 14 + .../range3d_rel_rang3d_encoding_random.rs | 19 + .../range3d_rel_range3d_encoding.rs | 24 ++ fuzz/src/encode.rs | 47 ++- 6 files changed, 509 insertions(+), 5 deletions(-) create mode 100644 fuzz/fuzz_targets/range3d_rel_rang3d_encoding_random.rs create mode 100644 fuzz/fuzz_targets/range3d_rel_range3d_encoding.rs diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 494b0b3..21df459 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -717,8 +717,6 @@ where header |= CompactWidth::from_u64(start_diff).bitmask(4); header |= CompactWidth::from_u64(end_diff).bitmask(6); - // println!("enc header: {:08b}", header); - consumer.consume(header).await?; match (&self.subspace, &out.subspace) { @@ -764,8 +762,6 @@ where { let header = produce_byte(producer).await?; - // println!("dec header: {:08b}", header); - // Decode subspace? let is_subspace_encoded = is_bitflagged(header, 0); @@ -910,3 +906,343 @@ where }) } } + +impl + RelativeEncodable> for Range3d +where + S: SubspaceId + Encodable + std::fmt::Debug, +{ + /// Encode an [`Range3d`] relative to another [`Range3d`] which [includes](https://willowprotocol.org/specs/grouping-entries/index.html#area_include_area) it. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_area_in_area). + async fn relative_encode( + &self, + reference: &Range3d, + consumer: &mut Consumer, + ) -> Result<(), Consumer::Error> + where + Consumer: BulkConsumer, + { + let start_to_start = self.times.start.abs_diff(reference.times.start); + let start_to_end = match reference.times.end { + RangeEnd::Closed(end) => self.times.start.abs_diff(end), + RangeEnd::Open => u64::MAX, + }; + let end_to_start = match self.times.end { + RangeEnd::Closed(end) => end.abs_diff(reference.times.start), + RangeEnd::Open => u64::MAX, + }; + let end_to_end = match (&self.times.end, &reference.times.end) { + (RangeEnd::Closed(self_end), RangeEnd::Closed(ref_end)) => self_end.abs_diff(*ref_end), + (RangeEnd::Closed(_), RangeEnd::Open) => u64::MAX, + (RangeEnd::Open, RangeEnd::Closed(_)) => u64::MAX, + (RangeEnd::Open, RangeEnd::Open) => 0, // shouldn't matter right??? + }; + + let start_time_diff = core::cmp::min(start_to_start, start_to_end); + + let end_time_diff = core::cmp::min(end_to_start, end_to_end); + + let mut header_1 = 0b0000_0000; + + // Bits 0, 1 - Encode r.subspaces.start? + if self.subspaces.start == reference.subspaces.start { + header_1 |= 0b0100_0000; + } else if reference.subspaces.end == self.subspaces.start { + header_1 |= 0b1000_0000; + } else { + header_1 |= 0b1100_0000; + } + + // Bits 2, 3 - Encode r.subspaces.end? + if self.subspaces.end == RangeEnd::Open { + // Do nothing + } else if self.subspaces.end == reference.subspaces.start { + header_1 |= 0b0001_0000; + } else if self.subspaces.end == reference.subspaces.end { + header_1 |= 0b0010_0000; + } else if self.subspaces.end != RangeEnd::Open { + header_1 |= 0b0011_0000; + } + + // Bit 4 - Encode r.paths.start relative to ref.paths.start or to ref.paths.end? + if let RangeEnd::Closed(ref_path_end) = &reference.paths.end { + let lcp_start_start = self + .paths + .start + .longest_common_prefix(&reference.paths.start); + let lcp_start_end = self.paths.start.longest_common_prefix(ref_path_end); + + if lcp_start_start.get_component_count() >= lcp_start_end.get_component_count() { + header_1 |= 0b0000_1000; + } + } else { + header_1 |= 0b0000_1000; + } + + // Bit 5 - Self path end open? + if self.paths.end == RangeEnd::Open { + header_1 |= 0b0000_0100; + } + + // Bit 6 - Encode r.paths.end relative to ref.paths.start or to ref.paths.end (if at all)? + match (&self.paths.end, &reference.paths.end) { + (RangeEnd::Closed(self_path_end), RangeEnd::Closed(ref_path_end)) => { + let lcp_end_start = self_path_end.longest_common_prefix(&reference.paths.start); + let lcp_end_end = self_path_end.longest_common_prefix(ref_path_end); + + if lcp_end_start.get_component_count() > lcp_end_end.get_component_count() { + header_1 |= 0b0000_0010; + } + } + (RangeEnd::Closed(_), RangeEnd::Open) => { + header_1 |= 0b0000_0010; + } + (RangeEnd::Open, RangeEnd::Closed(_)) => {} + (RangeEnd::Open, RangeEnd::Open) => {} + } + + // Bit 7 - Self time end open? + if self.times.end == RangeEnd::Open { + header_1 |= 0b0000_0001; + } + + consumer.consume(header_1).await?; + + let mut header_2 = 0b0000_0000; + + // Bit 8 - Encode r.times.start relative to ref.times.start or ref.times.end? + if start_to_start <= start_to_end { + header_2 |= 0b1000_0000; + } + + // Bit 9 -Add or subtract start_time_diff? + if is_bitflagged(header_2, 0) && self.times.start >= reference.times.start + || !is_bitflagged(header_2, 0) && self.times.start >= reference.times.end + { + header_2 |= 0b0100_0000; + } + + // Bit 10, 11 - 2-bit integer n such that 2^n gives compact_width(start_time_diff) + header_2 |= CompactWidth::from_u64(start_time_diff).bitmask(2); + + // Bit 12 - Encode r.times.end relative to ref.times.start or ref.times.end (if at all)? + if self.times.end != RangeEnd::Open && end_to_start <= end_to_end { + header_2 |= 0b0000_1000; + } + + // Bit 13 - Add or subtract end_time_diff (if encoding it at all)? + if self.times.end == RangeEnd::Open { + // do nothing + } else if (is_bitflagged(header_2, 4) && self.times.end >= reference.times.start) + || (!is_bitflagged(header_2, 4) && self.times.end >= reference.times.end) + { + header_2 |= 0b0000_0100; + } + + // Bits 14, 15 - ignored, or 2-bit integer n such that 2^n gives compact_width(end_time_diff) + if self.times.end == RangeEnd::Open { + // do nothing + } else { + header_2 |= CompactWidth::from_u64(end_time_diff).bitmask(6); + } + + consumer.consume(header_2).await?; + + if (self.subspaces.start == reference.subspaces.start) + || (reference.subspaces.end == self.subspaces.start) + { + // Don't encode + } else { + self.subspaces.start.encode(consumer).await?; + } + + if self.subspaces.end == RangeEnd::Open + || (self.subspaces.end == reference.subspaces.start) + || (self.subspaces.end == reference.subspaces.end) + { + // Don't encode end subspace + } else if let RangeEnd::Closed(end_subspace) = &self.subspaces.end { + end_subspace.encode(consumer).await?; + } + + if is_bitflagged(header_1, 4) { + self.paths + .start + .relative_encode(&reference.paths.start, consumer) + .await?; + } else if let RangeEnd::Closed(end_path) = &reference.paths.end { + self.paths.start.relative_encode(end_path, consumer).await?; + } + + if let RangeEnd::Closed(end_path) = &self.paths.end { + if is_bitflagged(header_1, 6) { + end_path + .relative_encode(&reference.paths.start, consumer) + .await? + } else if let RangeEnd::Closed(ref_end_path) = &reference.paths.end { + end_path.relative_encode(ref_end_path, consumer).await?; + } + } + + encode_compact_width_be(start_time_diff, consumer).await?; + encode_compact_width_be(end_time_diff, consumer).await?; + + Ok(()) + } +} + +impl + RelativeDecodable> for Range3d +where + S: SubspaceId + Decodable + std::fmt::Debug, +{ + /// Encode an [`Range3d`] relative to another [`Range3d`] which [includes](https://willowprotocol.org/specs/grouping-entries/index.html#area_include_area) it. + /// + /// [Definition](https://willowprotocol.org/specs/encodings/index.html#enc_area_in_area). + async fn relative_decode( + reference: &Range3d, + producer: &mut Producer, + ) -> Result> + where + Producer: BulkProducer, + Self: Sized, + { + let header_1 = produce_byte(producer).await?; + + let subspace_start_flags = header_1 & 0b1100_0000; + let subspace_end_flags = header_1 & 0b0011_0000; + let is_path_start_rel_to_start = is_bitflagged(header_1, 4); + let is_path_end_open = is_bitflagged(header_1, 5); + let is_path_end_rel_to_start = is_bitflagged(header_1, 6); + let is_times_end_open = is_bitflagged(header_1, 7); + + let header_2 = produce_byte(producer).await?; + + let is_time_start_rel_to_start = is_bitflagged(header_2, 0); + let add_or_subtract_start_time_diff = is_bitflagged(header_2, 1); + let start_time_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header_2, 2); + let is_times_end_rel_to_start = is_bitflagged(header_2, 4); + let add_or_subtract_end_time_diff = is_bitflagged(header_2, 5); + let end_time_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header_2, 6); + + // Decode subspace start + let subspace_start = match subspace_start_flags { + 0b0100_0000 => reference.subspaces.start.clone(), + 0b1000_0000 => match &reference.subspaces.end { + RangeEnd::Closed(end) => end.clone(), + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + }, + // This can only be 0b1100_0000 + _ => S::decode(producer).await?, + }; + + let subspace_end = match subspace_end_flags { + 0b0000_0000 => RangeEnd::Open, + 0b0001_0000 => RangeEnd::Closed(reference.subspaces.start.clone()), + 0b0010_0000 => match &reference.subspaces.end { + RangeEnd::Closed(end) => RangeEnd::Closed(end.clone()), + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + }, + // This can only be 0b0011_0000 + _ => RangeEnd::Closed(S::decode(producer).await?), + }; + + let path_start = match (is_path_start_rel_to_start, &reference.paths.end) { + (true, RangeEnd::Closed(_)) => { + Path::relative_decode(&reference.paths.start, producer).await? + } + (true, RangeEnd::Open) => { + Path::relative_decode(&reference.paths.start, producer).await? + } + (false, RangeEnd::Closed(path_end)) => { + Path::relative_decode(path_end, producer).await? + } + (false, RangeEnd::Open) => Err(DecodeError::InvalidInput)?, + }; + + let path_end = if is_path_end_open { + RangeEnd::Open + } else if is_path_end_rel_to_start { + RangeEnd::Closed(Path::relative_decode(&reference.paths.start, producer).await?) + } else { + match &reference.paths.end { + RangeEnd::Closed(end) => { + RangeEnd::Closed(Path::relative_decode(end, producer).await?) + } + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + } + }; + + let start_time_diff = + decode_compact_width_be(start_time_diff_compact_width, producer).await?; + + let time_start = match (is_time_start_rel_to_start, add_or_subtract_start_time_diff) { + (true, true) => reference.times.start.checked_add(start_time_diff), + (true, false) => reference.times.start.checked_sub(start_time_diff), + (false, true) => match reference.times.end { + RangeEnd::Closed(ref_end) => ref_end.checked_add(start_time_diff), + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + }, + (false, false) => match reference.times.end { + RangeEnd::Closed(ref_end) => ref_end.checked_sub(start_time_diff), + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + }, + } + .ok_or(DecodeError::InvalidInput)?; + + let end_time_diff = decode_compact_width_be(end_time_diff_compact_width, producer).await?; + + let time_end = if is_times_end_open { + RangeEnd::Open + } else { + match (is_times_end_rel_to_start, add_or_subtract_end_time_diff) { + (true, true) => RangeEnd::Closed( + reference + .times + .start + .checked_add(end_time_diff) + .ok_or(DecodeError::InvalidInput)?, + ), + (true, false) => RangeEnd::Closed( + reference + .times + .start + .checked_sub(end_time_diff) + .ok_or(DecodeError::InvalidInput)?, + ), + (false, true) => match reference.times.end { + RangeEnd::Closed(ref_end) => RangeEnd::Closed( + ref_end + .checked_add(end_time_diff) + .ok_or(DecodeError::InvalidInput)?, + ), + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + }, + (false, false) => match reference.times.end { + RangeEnd::Closed(ref_end) => RangeEnd::Closed( + ref_end + .checked_sub(end_time_diff) + .ok_or(DecodeError::InvalidInput)?, + ), + RangeEnd::Open => Err(DecodeError::InvalidInput)?, + }, + } + }; + + Ok(Self { + subspaces: Range { + start: subspace_start, + end: subspace_end, + }, + paths: Range { + start: path_start, + end: path_end, + }, + times: Range { + start: time_start, + end: time_end, + }, + }) + } +} diff --git a/data-model/src/grouping/range.rs b/data-model/src/grouping/range.rs index 759179a..c94c2f5 100644 --- a/data-model/src/grouping/range.rs +++ b/data-model/src/grouping/range.rs @@ -3,6 +3,8 @@ use core::cmp::Ordering; use arbitrary::{Arbitrary, Error as ArbitraryError}; +use crate::path::Path; + #[derive(Debug, PartialEq, Eq)] /// Determines whether a [`Range`] is _closed_ or _open_. pub enum RangeEnd { @@ -52,6 +54,70 @@ impl PartialOrd for RangeEnd { } } +impl PartialEq for RangeEnd +where + T: Eq + Ord, +{ + fn eq(&self, other: &T) -> bool { + match self { + RangeEnd::Closed(val) => val.eq(other), + RangeEnd::Open => false, + } + } +} + +impl PartialOrd for RangeEnd +where + T: Ord, +{ + fn partial_cmp(&self, other: &T) -> Option { + match self { + RangeEnd::Closed(val) => val.partial_cmp(other), + RangeEnd::Open => Some(Ordering::Greater), + } + } +} + +impl PartialEq> for u64 { + fn eq(&self, other: &RangeEnd) -> bool { + match other { + RangeEnd::Closed(other_val) => self.eq(other_val), + RangeEnd::Open => false, + } + } +} + +impl PartialOrd> for u64 { + fn partial_cmp(&self, other: &RangeEnd) -> Option { + match other { + RangeEnd::Closed(other_val) => self.partial_cmp(other_val), + RangeEnd::Open => Some(Ordering::Less), + } + } +} + +impl PartialEq>> + for Path +{ + fn eq(&self, other: &RangeEnd>) -> bool { + match other { + RangeEnd::Closed(other_path) => self.eq(other_path), + RangeEnd::Open => false, + } + } +} + +impl PartialOrd>> + for Path +{ + fn partial_cmp(&self, other: &RangeEnd>) -> Option { + match other { + RangeEnd::Closed(other_path) => self.partial_cmp(other_path), + RangeEnd::Open => Some(Ordering::Less), + } + } +} + #[cfg(feature = "dev")] impl<'a, T> Arbitrary<'a> for RangeEnd where diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e3b8eba..2c70738 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -245,3 +245,17 @@ path = "fuzz_targets/area_rel_area_encoding_random.rs" test = false doc = false bench = false + +[[bin]] +name = "range3d_rel_range3d_encoding" +path = "fuzz_targets/range3d_rel_range3d_encoding.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "range3d_rel_rang3d_encoding_random" +path = "fuzz_targets/range3d_rel_rang3d_encoding_random.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/range3d_rel_rang3d_encoding_random.rs b/fuzz/fuzz_targets/range3d_rel_rang3d_encoding_random.rs new file mode 100644 index 0000000..5eb99f0 --- /dev/null +++ b/fuzz/fuzz_targets/range3d_rel_rang3d_encoding_random.rs @@ -0,0 +1,19 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use libfuzzer_sys::fuzz_target; +use willow_data_model::grouping::range_3d::Range3d; +use willow_data_model_fuzz::encode::relative_encoding_random_less_strict; + +fuzz_target!(|data: (&[u8], Range3d<16, 16, 16, IdentityId>)| { + // fuzzed code goes here + let (random_bytes, area) = data; + + smol::block_on(async { + relative_encoding_random_less_strict::< + Range3d<16, 16, 16, IdentityId>, + Range3d<16, 16, 16, IdentityId>, + >(area, random_bytes) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/range3d_rel_range3d_encoding.rs b/fuzz/fuzz_targets/range3d_rel_range3d_encoding.rs new file mode 100644 index 0000000..aaf80bd --- /dev/null +++ b/fuzz/fuzz_targets/range3d_rel_range3d_encoding.rs @@ -0,0 +1,24 @@ +#![no_main] + +use earthstar::identity_id::IdentityIdentifier as IdentityId; +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use willow_data_model::grouping::range_3d::Range3d; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; + +fuzz_target!(|data: ( + Range3d<16, 16, 16, IdentityId>, + Range3d<16, 16, 16, IdentityId>, + TestConsumer +)| { + let (ran, reference, mut consumer) = data; + + smol::block_on(async { + relative_encoding_roundtrip::< + Range3d<16, 16, 16, IdentityId>, + Range3d<16, 16, 16, IdentityId>, + TestConsumer, + >(ran, reference, &mut consumer) + .await; + }); +}); diff --git a/fuzz/src/encode.rs b/fuzz/src/encode.rs index ac47296..a5b4def 100644 --- a/fuzz/src/encode.rs +++ b/fuzz/src/encode.rs @@ -79,7 +79,7 @@ pub async fn relative_encoding_roundtrip( R: std::fmt::Debug, C: BulkConsumer, { - // println!("item {:?}", subject); + //println!("item {:?}", subject); // println!("ref {:?}", reference); if let Err(_err) = subject.relative_encode(&reference, consumer).await { @@ -139,3 +139,48 @@ where }, }; } + +pub async fn relative_encoding_random_less_strict(reference: R, data: &[u8]) +where + T: RelativeEncodable + RelativeDecodable + std::fmt::Debug + Eq, + R: std::fmt::Debug, +{ + let mut producer = FromSlice::new(data); + + match T::relative_decode(&reference, &mut producer).await { + Ok(item) => { + // It decoded to a valid item! Gasp! + // Can we turn it back into the same encoding? + let mut consumer = IntoVec::::new(); + + // println!("item {:?}", item); + // println!("ref {:?}", reference); + + item.relative_encode(&reference, &mut consumer) + .await + .unwrap(); + + let mut producer_2 = FromSlice::new(consumer.as_ref()); + + match T::relative_decode(&reference, &mut producer_2).await { + Ok(decoded_again) => { + assert_eq!(item, decoded_again); + } + Err(err) => { + println!("{:?}", err); + panic!("Could not decode again, argh!") + } + } + } + Err(err) => match err { + // There was an error. + DecodeError::Producer(_) => panic!("Returned producer error, when whe shouldn't!"), + DecodeError::InvalidInput => { + // GOOD. + } + DecodeError::U64DoesNotFitUsize => { + panic!("Returned u64DoesNotFitUsize error, when we shouldn't!") + } + }, + }; +} From a5cafd97c0595313adb86891bc6c8b80595fac1f Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 22 Jul 2024 17:01:26 +0100 Subject: [PATCH 28/40] Make all relative encoders a bit less strict --- data-model/src/encoding/relativity.rs | 30 +++++++++++++++++-- .../area_rel_area_encoding_random.rs | 10 +++---- .../entry_rel_entry_encoding_random.rs | 4 +-- ...ntry_rel_namespace_area_encoding_random.rs | 4 +-- ...try_rel_namespace_range_encoding_random.rs | 4 +-- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 21df459..f4938b5 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -228,11 +228,13 @@ where reference.namespace_id.clone() }; + /* // Verify that the encoded namespace wasn't the same as ours // Which would indicate invalid input if is_namespace_encoded && namespace_id == reference.namespace_id { return Err(DecodeError::InvalidInput); } + */ let subspace_id = if is_subspace_encoded { S::decode(producer).await? @@ -240,11 +242,13 @@ where reference.subspace_id.clone() }; + /* // Verify that the encoded subspace wasn't the same as ours // Which would indicate invalid input if is_subspace_encoded && subspace_id == reference.subspace_id { return Err(DecodeError::InvalidInput); } + */ let path = Path::::relative_decode(&reference.path, producer).await?; @@ -263,11 +267,13 @@ where .ok_or(DecodeError::InvalidInput)? }; + /* // Verify that the correct add_or_subtract_time_diff flag was set. let should_have_subtracted = timestamp <= reference.timestamp; if add_or_subtract_time_diff && should_have_subtracted { return Err(DecodeError::InvalidInput); } + */ let payload_length = decode_compact_width_be(compact_width_payload_length, producer).await?; @@ -408,6 +414,7 @@ where } .ok_or(DecodeError::InvalidInput)?; + /* // Verify that the correct add_or_subtract_time_diff flag was set. let should_have_added = timestamp.checked_sub(out.times.start) <= u64::from(&out.times.end).checked_sub(timestamp); @@ -415,6 +422,7 @@ where if add_time_diff_to_start != should_have_added { return Err(DecodeError::InvalidInput); } + */ if !out.times.includes(×tamp) { return Err(DecodeError::InvalidInput); @@ -575,10 +583,12 @@ where out.subspaces.start.clone() }; + /* // Verify that encoding the subspace was necessary. if subspace_id == out.subspaces.start && is_subspace_encoded { return Err(DecodeError::InvalidInput); } + */ // Verify that subspace is included by range if !out.subspaces.includes(&subspace_id) { @@ -599,6 +609,7 @@ where return Err(DecodeError::InvalidInput); } + /* // Verify that the path was encoded relative to the correct bound of the referenc path range. let should_have_encoded_path_relative_to_start = match &out.paths.end { RangeEnd::Closed(end_path) => { @@ -610,9 +621,11 @@ where RangeEnd::Open => true, }; + if decode_path_relative_to_start != should_have_encoded_path_relative_to_start { return Err(DecodeError::InvalidInput); } + */ let time_diff = decode_compact_width_be(time_diff_compact_width, producer).await?; @@ -636,6 +649,7 @@ where return Err(DecodeError::InvalidInput); } + /* // Verify that time_diff is what it should have been let correct_time_diff = core::cmp::min( timestamp.abs_diff(out.times.start), @@ -646,12 +660,14 @@ where return Err(DecodeError::InvalidInput); } + // Verify that the combine with start bitflag in the header was correct let should_have_added_to_start = time_diff == timestamp.abs_diff(out.times.start); if should_have_added_to_start != add_time_diff_with_start { return Err(DecodeError::InvalidInput); } + */ Ok(Self { namespace_id: namespace.clone(), @@ -774,30 +790,34 @@ where // Add end_diff to out.times.start, or subtract from out.times.end? let add_end_diff = is_bitflagged(header, 3); + /* // Verify that we don't add_end_diff when open... - if add_end_diff && is_times_end_open { return Err(DecodeError::InvalidInput); } + */ let start_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 4); let end_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 6); + /* // Verify the last two bits are zero if is_times_end_open if is_times_end_open && (end_diff_compact_width != CompactWidth::One) { return Err(DecodeError::InvalidInput); } + */ let subspace = if is_subspace_encoded { let id = S::decode(producer).await?; - let sub = AreaSubspace::Id(id); + AreaSubspace::Id(id) + /* // Verify that subspace wasn't needlessly encoded if sub == out.subspace { return Err(DecodeError::InvalidInput); } - sub + */ } else { out.subspace.clone() }; @@ -843,6 +863,7 @@ where return Err(DecodeError::InvalidInput); } + /* // Verify that bit 2 of the header was set correctly let should_add_start_diff = start_diff == start @@ -852,6 +873,7 @@ where if add_start_diff != should_add_start_diff { return Err(DecodeError::InvalidInput); } + */ let end = if is_times_end_open { if add_end_diff { @@ -880,6 +902,7 @@ where return Err(DecodeError::InvalidInput); } + /* let should_add_end_diff = end_diff == end .checked_sub(out.times.start) @@ -888,6 +911,7 @@ where if add_end_diff != should_add_end_diff { return Err(DecodeError::InvalidInput); } + */ RangeEnd::Closed(end) }; diff --git a/fuzz/fuzz_targets/area_rel_area_encoding_random.rs b/fuzz/fuzz_targets/area_rel_area_encoding_random.rs index a2f9917..9f632c8 100644 --- a/fuzz/fuzz_targets/area_rel_area_encoding_random.rs +++ b/fuzz/fuzz_targets/area_rel_area_encoding_random.rs @@ -3,17 +3,17 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use libfuzzer_sys::fuzz_target; use willow_data_model::grouping::area::Area; -use willow_data_model_fuzz::encode::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random_less_strict; fuzz_target!(|data: (&[u8], Area<16, 16, 16, IdentityId>)| { // fuzzed code goes here let (random_bytes, area) = data; smol::block_on(async { - relative_encoding_random::, Area<16, 16, 16, IdentityId>>( - area, - random_bytes, - ) + relative_encoding_random_less_strict::< + Area<16, 16, 16, IdentityId>, + Area<16, 16, 16, IdentityId>, + >(area, random_bytes) .await; }); }); diff --git a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs index a110ef7..a905b99 100644 --- a/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_entry_encoding_random.rs @@ -4,7 +4,7 @@ use earthstar::identity_id::IdentityIdentifier as IdentityId; use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; use libfuzzer_sys::fuzz_target; use willow_data_model::entry::Entry; -use willow_data_model_fuzz::encode::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random_less_strict; use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!(|data: ( @@ -14,7 +14,7 @@ fuzz_target!(|data: ( let (random_bytes, ref_entry) = data; smol::block_on(async { - relative_encoding_random::< + relative_encoding_random_less_strict::< Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, >(ref_entry, random_bytes) diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs index 6272a8b..36381ec 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding_random.rs @@ -5,7 +5,7 @@ use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; use libfuzzer_sys::fuzz_target; use willow_data_model::entry::Entry; use willow_data_model::grouping::area::Area; -use willow_data_model_fuzz::encode::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random_less_strict; use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!( @@ -14,7 +14,7 @@ fuzz_target!( let (random_bytes, namespaced_area) = data; smol::block_on(async { - relative_encoding_random::< + relative_encoding_random_less_strict::< (EsNamespaceId, Area<16, 16, 16, IdentityId>), Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, >(namespaced_area, random_bytes) diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs index cc0e69b..1801a81 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding_random.rs @@ -5,7 +5,7 @@ use earthstar::namespace_id::NamespaceIdentifier as EsNamespaceId; use libfuzzer_sys::fuzz_target; use willow_data_model::entry::Entry; use willow_data_model::grouping::range_3d::Range3d; -use willow_data_model_fuzz::encode::relative_encoding_random; +use willow_data_model_fuzz::encode::relative_encoding_random_less_strict; use willow_data_model_fuzz::placeholder_params::FakePayloadDigest; fuzz_target!( @@ -14,7 +14,7 @@ fuzz_target!( let (random_bytes, namespaced_range_3d) = data; smol::block_on(async { - relative_encoding_random::< + relative_encoding_random_less_strict::< (EsNamespaceId, Range3d<16, 16, 16, IdentityId>), Entry<16, 16, 16, EsNamespaceId, IdentityId, FakePayloadDigest>, >(namespaced_range_3d, random_bytes) From b051ae076b3631f6acffc8ce50c0ec6d375494b3 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 23 Jul 2024 07:37:22 +0100 Subject: [PATCH 29/40] Uncomment checks for canonic relative encodings --- data-model/src/encoding/relativity.rs | 45 ++++++++++++++------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index f4938b5..237a6fc 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -19,7 +19,8 @@ use crate::{ path::Path, }; -/// A type that can be used to encoded to a bytestring *encoded relative to `R`*. +/// A type that can be used to encode `T` to a bytestring *encoded relative to `R`*. +/// This can be used to create more compact encodings from which `T` can be derived by anyone with `R`. pub trait RelativeEncodable { /// A function from the set `Self` to the set of bytestrings *encoded relative to `reference`*. fn relative_encode( @@ -32,6 +33,7 @@ pub trait RelativeEncodable { } /// A type that can be used to decode `T` from a bytestring *encoded relative to `Self`*. +/// This can be used to decode a compact encoding frow which `T` can be derived by anyone with `R`. pub trait RelativeDecodable { /// A function from the set of bytestrings *encoded relative to `Self`* to the set of `T` in relation to `Self`. fn relative_decode( @@ -414,7 +416,7 @@ where } .ok_or(DecodeError::InvalidInput)?; - /* + // === Necessary to produce canonic encodings. === // Verify that the correct add_or_subtract_time_diff flag was set. let should_have_added = timestamp.checked_sub(out.times.start) <= u64::from(&out.times.end).checked_sub(timestamp); @@ -422,7 +424,7 @@ where if add_time_diff_to_start != should_have_added { return Err(DecodeError::InvalidInput); } - */ + // =============================================== if !out.times.includes(×tamp) { return Err(DecodeError::InvalidInput); @@ -583,12 +585,12 @@ where out.subspaces.start.clone() }; - /* + // === Necessary to produce canonic encodings. === // Verify that encoding the subspace was necessary. if subspace_id == out.subspaces.start && is_subspace_encoded { return Err(DecodeError::InvalidInput); } - */ + // =============================================== // Verify that subspace is included by range if !out.subspaces.includes(&subspace_id) { @@ -609,7 +611,7 @@ where return Err(DecodeError::InvalidInput); } - /* + // === Necessary to produce canonic encodings. === // Verify that the path was encoded relative to the correct bound of the referenc path range. let should_have_encoded_path_relative_to_start = match &out.paths.end { RangeEnd::Closed(end_path) => { @@ -621,11 +623,10 @@ where RangeEnd::Open => true, }; - if decode_path_relative_to_start != should_have_encoded_path_relative_to_start { return Err(DecodeError::InvalidInput); } - */ + // ================================================= let time_diff = decode_compact_width_be(time_diff_compact_width, producer).await?; @@ -649,7 +650,7 @@ where return Err(DecodeError::InvalidInput); } - /* + // === Necessary to produce canonic encodings. === // Verify that time_diff is what it should have been let correct_time_diff = core::cmp::min( timestamp.abs_diff(out.times.start), @@ -660,14 +661,13 @@ where return Err(DecodeError::InvalidInput); } - // Verify that the combine with start bitflag in the header was correct let should_have_added_to_start = time_diff == timestamp.abs_diff(out.times.start); if should_have_added_to_start != add_time_diff_with_start { return Err(DecodeError::InvalidInput); } - */ + // ============================================== Ok(Self { namespace_id: namespace.clone(), @@ -790,34 +790,35 @@ where // Add end_diff to out.times.start, or subtract from out.times.end? let add_end_diff = is_bitflagged(header, 3); - /* + // === Necessary to produce canonic encodings. === // Verify that we don't add_end_diff when open... if add_end_diff && is_times_end_open { return Err(DecodeError::InvalidInput); } - */ + // =============================================== let start_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 4); let end_diff_compact_width = CompactWidth::decode_fixed_width_bitmask(header, 6); - /* + // === Necessary to produce canonic encodings. === // Verify the last two bits are zero if is_times_end_open if is_times_end_open && (end_diff_compact_width != CompactWidth::One) { return Err(DecodeError::InvalidInput); } - */ + // =============================================== let subspace = if is_subspace_encoded { let id = S::decode(producer).await?; - AreaSubspace::Id(id) + let sub = AreaSubspace::Id(id); - /* + // === Necessary to produce canonic encodings. === // Verify that subspace wasn't needlessly encoded if sub == out.subspace { return Err(DecodeError::InvalidInput); } + // =============================================== - */ + sub } else { out.subspace.clone() }; @@ -863,7 +864,7 @@ where return Err(DecodeError::InvalidInput); } - /* + // === Necessary to produce canonic encodings. === // Verify that bit 2 of the header was set correctly let should_add_start_diff = start_diff == start @@ -873,7 +874,7 @@ where if add_start_diff != should_add_start_diff { return Err(DecodeError::InvalidInput); } - */ + // =============================================== let end = if is_times_end_open { if add_end_diff { @@ -902,7 +903,7 @@ where return Err(DecodeError::InvalidInput); } - /* + // === Necessary to produce canonic encodings. === let should_add_end_diff = end_diff == end .checked_sub(out.times.start) @@ -911,7 +912,7 @@ where if add_end_diff != should_add_end_diff { return Err(DecodeError::InvalidInput); } - */ + // ============================================ RangeEnd::Closed(end) }; From e5258281f1448a5ee3752eac57a7605c9a34d929 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 00:25:10 +0200 Subject: [PATCH 30/40] Implement more efficient decoders --- data-model/src/encoding/mod.rs | 1 + data-model/src/encoding/relativity.rs | 92 ++++++++++++--- data-model/src/encoding/shared_buffers.rs | 137 ++++++++++++++++++++++ data-model/src/lib.rs | 9 +- data-model/src/path.rs | 44 ++++--- fuzz/fuzz_targets/path_encoding.rs | 6 +- 6 files changed, 255 insertions(+), 34 deletions(-) create mode 100644 data-model/src/encoding/shared_buffers.rs diff --git a/data-model/src/encoding/mod.rs b/data-model/src/encoding/mod.rs index a1c1717..ad00642 100644 --- a/data-model/src/encoding/mod.rs +++ b/data-model/src/encoding/mod.rs @@ -4,4 +4,5 @@ pub mod error; pub mod max_power; pub mod parameters; pub mod relativity; +pub(crate) mod shared_buffers; pub mod unsigned_int; diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 59bdc59..a641621 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -1,4 +1,5 @@ use core::future::Future; +use core::mem::{size_of, MaybeUninit}; use ufotofu::local_nb::{BulkConsumer, BulkProducer}; use crate::{ @@ -19,6 +20,8 @@ use crate::{ path::Path, }; +use super::shared_buffers::ScratchSpacePathDecoding; + /// A type that can be used to encoded to a bytestring *encoded relative to `R`*. pub trait RelativeEncodable { /// A function from the set `Self` to the set of bytestrings *encoded relative to `reference`*. @@ -62,10 +65,9 @@ impl RelativeEncodable RelativeDecodable, Self: Sized, { - let lcp = decode_max_power(MCC, producer).await?; + let lcp_component_count: usize = decode_max_power(MCC, producer).await?.try_into()?; - if lcp > reference.get_component_count() as u64 { - return Err(DecodeError::InvalidInput); + if lcp_component_count == 0 { + let decoded = Path::::decode(producer).await?; + + // === Necessary to produce canonic encodings. === + if lcp_component_count + != decoded + .longest_common_prefix(reference) + .get_component_count() + { + return Err(DecodeError::InvalidInput); + } + // =============================================== + + return Ok(decoded); } let prefix = reference - .create_prefix(lcp as usize) + .create_prefix(lcp_component_count as usize) .ok_or(DecodeError::InvalidInput)?; - let suffix = Path::::decode(producer).await?; - let mut new = prefix; + let mut buf = ScratchSpacePathDecoding::::new(); + + // Copy the accumulated component lengths of the prefix into the scratch buffer. + let raw_prefix_acc_component_lengths = + &prefix.raw_buf()[size_of::()..size_of::() * (lcp_component_count + 1)]; + unsafe { + // Safe because len is less than size_of::() times the MCC, because `prefix` respects the MCC. + buf.set_many_component_accumulated_lengths_from_ne(raw_prefix_acc_component_lengths); + } + + // Copy the raw path data of the prefix into the scratch buffer. + unsafe { + // safe because we just copied the accumulated component lengths for the first `lcp_component_count` components. + MaybeUninit::copy_from_slice( + buf.path_data_until_as_mut(lcp_component_count), + &reference.raw_buf()[size_of::() * (reference.get_component_count() + 1) + ..size_of::() * (reference.get_component_count() + 1) + + prefix.get_path_length()], + ); + } + + let remaining_component_count: usize = decode_max_power(MCC, producer).await?.try_into()?; + let total_component_count = lcp_component_count + remaining_component_count; + if total_component_count > MCC { + return Err(DecodeError::InvalidInput); + } - for component in suffix.components() { - match new.append(component) { - Ok(appended) => new = appended, - Err(_) => return Err(DecodeError::InvalidInput), + let mut accumulated_component_length: usize = prefix.get_path_length(); // Always holds the acc length of all components we copied so far. + for i in lcp_component_count..total_component_count { + let component_len: usize = decode_max_power(MCL, producer).await?.try_into()?; + if component_len > MCL { + return Err(DecodeError::InvalidInput); } + + accumulated_component_length += component_len; + if accumulated_component_length > MPL { + return Err(DecodeError::InvalidInput); + } + + buf.set_component_accumulated_length(accumulated_component_length, i); + + // Decode the component itself into the scratch buffer. + producer + .bulk_overwrite_full_slice_uninit(unsafe { + // Safe because we called set_component_Accumulated_length for all j <= i + buf.path_data_as_mut(i) + }) + .await?; } - let actual_lcp = reference.longest_common_prefix(&new); + let decoded = unsafe { buf.to_path(total_component_count) }; - if actual_lcp.get_component_count() != lcp as usize { + // === Necessary to produce canonic encodings. === + if lcp_component_count + != decoded + .longest_common_prefix(reference) + .get_component_count() + { return Err(DecodeError::InvalidInput); } + // =============================================== - Ok(new) + return Ok(decoded); } } diff --git a/data-model/src/encoding/shared_buffers.rs b/data-model/src/encoding/shared_buffers.rs new file mode 100644 index 0000000..5b97569 --- /dev/null +++ b/data-model/src/encoding/shared_buffers.rs @@ -0,0 +1,137 @@ +//! Shared buffers to be reused across many decoding operations. + +use core::mem::MaybeUninit; + +use bytes::BytesMut; + +use crate::path::Path; + +/// A memory region to use for decoding paths. Reused between many decodings. +#[derive(Debug)] +pub(crate) struct ScratchSpacePathDecoding { + // The i-th usize holds the total lengths of the first i components. + component_accumulated_lengths: [MaybeUninit; MCC], + path_data: [MaybeUninit; MPL], +} + +impl ScratchSpacePathDecoding { + pub fn new() -> Self { + ScratchSpacePathDecoding { + component_accumulated_lengths: MaybeUninit::uninit_array(), + path_data: MaybeUninit::uninit_array(), + } + } + + /// Panic if i >= MCC. + pub fn set_component_accumulated_length( + &mut self, + component_accumulated_length: usize, + i: usize, + ) { + MaybeUninit::write( + &mut self.component_accumulated_lengths[i], + component_accumulated_length, + ); + } + + /// # Saftey + /// + /// UB if length of slice is greater than `size_of::() * MCC`. + pub unsafe fn set_many_component_accumulated_lengths_from_ne( + &mut self, + lengths: &[u8], + ) { + let slice: &mut [MaybeUninit] = core::slice::from_raw_parts_mut( + self.component_accumulated_lengths[..lengths.len() / size_of::()].as_mut_ptr() as *mut MaybeUninit, + lengths.len(), + ); + MaybeUninit::copy_from_slice( + slice, + lengths + ); + } + + /// # Safety + /// + /// Memory must have been initialised with a prior call to set_component_accumulated_length for the same `i`. + unsafe fn get_component_accumulated_length(&self, i: usize) -> usize { + return MaybeUninit::assume_init(self.component_accumulated_lengths[i]); + } + + /// Return a slice of the accumulated component lengths up to but excluding the `i`-th component, encoded as native-endian u8s. + /// + /// # Safety + /// + /// Memory must have been initialised with prior call to set_component_accumulated_length for all `j <= i` + pub unsafe fn get_accumumulated_component_lengths(&self, i: usize) -> &[u8] { + core::slice::from_raw_parts( + MaybeUninit::slice_assume_init_ref(&self.component_accumulated_lengths[..i]).as_ptr() + as *const u8, + i * size_of::(), + ) + } + + /// Return a mutable slice of the i-th path_data. + /// + /// # Safety + /// + /// Accumulated component lengths for `i` and `i - 1` must have been set (only for `i` if `i == 0`). + pub unsafe fn path_data_as_mut(&mut self, i: usize) -> &mut [MaybeUninit] { + let start = if i == 0 { + 0 + } else { + self.get_component_accumulated_length(i - 1) + }; + let end = self.get_component_accumulated_length(i); + return &mut self.path_data[start..end]; + } + + /// Return a mutable slice of the path_data up to but excluding the i-th component. + /// + /// # Safety + /// + /// Accumulated component lengths for `i - 1` must have been set (unless `i == 0`). + pub unsafe fn path_data_until_as_mut(&mut self, i: usize) -> &mut [MaybeUninit] { + let end = self.get_component_accumulated_length(i - 1); + return &mut self.path_data[0..end]; + } + + /// Get the path data of the first `i` components. + /// + /// # Safety + /// + /// Memory must have been initialised with a prior call to set_component_accumulated_length for `i - 1` (ignored if `i == 0`). + /// Also, the path data must have been initialised via a reference obtained from `self.path_data_as_mut()`. + unsafe fn get_path_data(&self, i: usize) -> &[u8] { + let end = if i == 0 { + 0 + } else { + self.get_component_accumulated_length(i - 1) + }; + return MaybeUninit::slice_assume_init_ref(&self.path_data[..end]); + } + + /// Copy the data from this struct into a new Path of `i` components. + /// + /// # Safety + /// + /// The first `i` accumulated component lengths must have been set, and all corresponding path data must be initialised. MCL, MCC, and MCP are trusted blindly and must be adhered to by the data in the scratch buffer. + pub unsafe fn to_path(&self, i: usize) -> Path { + if i == 0 { + return Path::new_empty(); + } else { + let total_length = if i == 0 { + 0 + } else { + self.get_component_accumulated_length(i - 1) + }; + let mut buf = BytesMut::with_capacity((size_of::() * (i + 1)) + total_length); + + buf.extend_from_slice(&i.to_ne_bytes()); + buf.extend_from_slice(self.get_accumumulated_component_lengths(i)); + buf.extend_from_slice(self.get_path_data(i)); + + return Path::from_buffer_and_component_count(buf.freeze(), i); + } + } +} diff --git a/data-model/src/lib.rs b/data-model/src/lib.rs index 113acc6..c665034 100644 --- a/data-model/src/lib.rs +++ b/data-model/src/lib.rs @@ -1,4 +1,11 @@ -#![feature(new_uninit, async_fn_traits, debug_closure_helpers)] +#![feature( + new_uninit, + async_fn_traits, + debug_closure_helpers, + maybe_uninit_uninit_array, + maybe_uninit_write_slice, + maybe_uninit_slice +)] pub mod encoding; pub mod entry; diff --git a/data-model/src/path.rs b/data-model/src/path.rs index 42d17e1..3bbe3da 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -6,6 +6,7 @@ use crate::encoding::{ error::DecodeError, max_power::{decode_max_power, encode_max_power}, parameters::{Decodable, Encodable}, + shared_buffers::ScratchSpacePathDecoding, }; // This struct is tested in `fuzz/path.rs`, `fuzz/path2.rs`, `fuzz/path3.rs`, `fuzz/path3.rs` by comparing against a non-optimised reference implementation. @@ -660,12 +661,16 @@ impl Path { None } - fn from_buffer_and_component_count(buf: Bytes, component_count: usize) -> Self { + pub(crate) fn from_buffer_and_component_count(buf: Bytes, component_count: usize) -> Self { Path { data: HeapEncoding(buf), component_count, } } + + pub(crate) fn raw_buf(&self) -> &[u8] { + self.data.0.as_ref() + } } impl PartialEq for Path { @@ -808,26 +813,37 @@ impl Decodable for Path, { - let component_count = decode_max_power(MCC, producer).await?; + let component_count: usize = decode_max_power(MCC, producer).await?.try_into()?; + if component_count > MCC { + return Err(DecodeError::InvalidInput); + } - let mut path = Self::new_empty(); + let mut buf = ScratchSpacePathDecoding::::new(); - for _ in 0..component_count { - let component_len = decode_max_power(MCL, producer).await?; + let mut accumulated_component_length: usize = 0; // Always holds the acc length of all components we copied so far. + for i in 0..component_count { + let component_len: usize = decode_max_power(MCL, producer).await?.try_into()?; + if component_len > MCL { + return Err(DecodeError::InvalidInput); + } + + accumulated_component_length += component_len; + if accumulated_component_length > MPL { + return Err(DecodeError::InvalidInput); + } - let mut component_box = Box::new_uninit_slice(usize::try_from(component_len)?); + buf.set_component_accumulated_length(accumulated_component_length, i); - let slice = producer - .bulk_overwrite_full_slice_uninit(component_box.as_mut()) + // Decode the component itself into the scratch buffer. + producer + .bulk_overwrite_full_slice_uninit(unsafe { + // Safe because we called set_component_Accumulated_length for all j <= i + buf.path_data_as_mut(i) + }) .await?; - - let path_component = Component::new(slice).ok_or(DecodeError::InvalidInput)?; - path = path - .append(path_component) - .map_err(|_| DecodeError::InvalidInput)?; } - Ok(path) + return Ok(unsafe { buf.to_path(component_count) }); } } diff --git a/fuzz/fuzz_targets/path_encoding.rs b/fuzz/fuzz_targets/path_encoding.rs index ad4f234..6013aa0 100644 --- a/fuzz/fuzz_targets/path_encoding.rs +++ b/fuzz/fuzz_targets/path_encoding.rs @@ -5,9 +5,9 @@ use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::path::Path; use willow_data_model_fuzz::encode::encoding_roundtrip; -const MCL: usize = 111111; -const MCC: usize = 111111; -const MPL: usize = 111111; +const MCL: usize = 300; +const MCC: usize = 300; +const MPL: usize = 300; fuzz_target!(|data: (Path, TestConsumer)| { let (path, mut consumer) = data; From 5d98f3157655bef2a2c465243b9d3741f45e2823 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 00:39:11 +0200 Subject: [PATCH 31/40] Add more path rel path fuzz tests --- fuzz/Cargo.toml | 56 +++++++++++++++++++ fuzz/fuzz_targets/path_encoding2.rs | 18 ++++++ fuzz/fuzz_targets/path_encoding3.rs | 18 ++++++ fuzz/fuzz_targets/path_encoding_random.rs | 6 +- fuzz/fuzz_targets/path_encoding_random2.rs | 15 +++++ fuzz/fuzz_targets/path_encoding_random3.rs | 15 +++++ fuzz/fuzz_targets/path_rel_path_encoding.rs | 6 +- fuzz/fuzz_targets/path_rel_path_encoding2.rs | 28 ++++++++++ fuzz/fuzz_targets/path_rel_path_encoding3.rs | 28 ++++++++++ .../path_rel_path_encoding_random.rs | 6 +- .../path_rel_path_encoding_random2.rs | 21 +++++++ .../path_rel_path_encoding_random3.rs | 21 +++++++ 12 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 fuzz/fuzz_targets/path_encoding2.rs create mode 100644 fuzz/fuzz_targets/path_encoding3.rs create mode 100644 fuzz/fuzz_targets/path_encoding_random2.rs create mode 100644 fuzz/fuzz_targets/path_encoding_random3.rs create mode 100644 fuzz/fuzz_targets/path_rel_path_encoding2.rs create mode 100644 fuzz/fuzz_targets/path_rel_path_encoding3.rs create mode 100644 fuzz/fuzz_targets/path_rel_path_encoding_random2.rs create mode 100644 fuzz/fuzz_targets/path_rel_path_encoding_random3.rs diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e3b8eba..13c9fce 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -99,6 +99,20 @@ test = false doc = false bench = false +[[bin]] +name = "path_encoding2" +path = "fuzz_targets/path_encoding2.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "path_encoding3" +path = "fuzz_targets/path_encoding3.rs" +test = false +doc = false +bench = false + [[bin]] name = "path_encoding_random" path = "fuzz_targets/path_encoding_random.rs" @@ -106,6 +120,20 @@ test = false doc = false bench = false +[[bin]] +name = "path_encoding_random2" +path = "fuzz_targets/path_encoding_random2.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "path_encoding_random3" +path = "fuzz_targets/path_encoding_random3.rs" +test = false +doc = false +bench = false + [[bin]] name = "entry_encoding" path = "fuzz_targets/entry_encoding.rs" @@ -183,6 +211,20 @@ test = false doc = false bench = false +[[bin]] +name = "path_rel_path_encoding2" +path = "fuzz_targets/path_rel_path_encoding2.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "path_rel_path_encoding3" +path = "fuzz_targets/path_rel_path_encoding3.rs" +test = false +doc = false +bench = false + [[bin]] name = "path_rel_path_encoding_random" path = "fuzz_targets/path_rel_path_encoding_random.rs" @@ -190,6 +232,20 @@ test = false doc = false bench = false +[[bin]] +name = "path_rel_path_encoding_random2" +path = "fuzz_targets/path_rel_path_encoding_random2.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "path_rel_path_encoding_random3" +path = "fuzz_targets/path_rel_path_encoding_random3.rs" +test = false +doc = false +bench = false + [[bin]] name = "entry_rel_entry_encoding" path = "fuzz_targets/entry_rel_entry_encoding.rs" diff --git a/fuzz/fuzz_targets/path_encoding2.rs b/fuzz/fuzz_targets/path_encoding2.rs new file mode 100644 index 0000000..3147686 --- /dev/null +++ b/fuzz/fuzz_targets/path_encoding2.rs @@ -0,0 +1,18 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::encoding_roundtrip; + +const MCL: usize = 2; +const MCC: usize = 3; +const MPL: usize = 3; + +fuzz_target!(|data: (Path, TestConsumer)| { + let (path, mut consumer) = data; + + smol::block_on(async { + encoding_roundtrip::<_, TestConsumer>(path, &mut consumer).await; + }); +}); diff --git a/fuzz/fuzz_targets/path_encoding3.rs b/fuzz/fuzz_targets/path_encoding3.rs new file mode 100644 index 0000000..45ecabf --- /dev/null +++ b/fuzz/fuzz_targets/path_encoding3.rs @@ -0,0 +1,18 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::encoding_roundtrip; + +const MCL: usize = 4; +const MCC: usize = 4; +const MPL: usize = 16; + +fuzz_target!(|data: (Path, TestConsumer)| { + let (path, mut consumer) = data; + + smol::block_on(async { + encoding_roundtrip::<_, TestConsumer>(path, &mut consumer).await; + }); +}); diff --git a/fuzz/fuzz_targets/path_encoding_random.rs b/fuzz/fuzz_targets/path_encoding_random.rs index 26d980b..40507c5 100644 --- a/fuzz/fuzz_targets/path_encoding_random.rs +++ b/fuzz/fuzz_targets/path_encoding_random.rs @@ -4,9 +4,9 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::path::Path; use willow_data_model_fuzz::encode::encoding_random; -const MCL: usize = 111111; -const MCC: usize = 111111; -const MPL: usize = 111111; +const MCL: usize = 300; +const MCC: usize = 300; +const MPL: usize = 300; fuzz_target!(|data: &[u8]| { smol::block_on(async { diff --git a/fuzz/fuzz_targets/path_encoding_random2.rs b/fuzz/fuzz_targets/path_encoding_random2.rs new file mode 100644 index 0000000..b83d37c --- /dev/null +++ b/fuzz/fuzz_targets/path_encoding_random2.rs @@ -0,0 +1,15 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::encoding_random; + +const MCL: usize = 2; +const MCC: usize = 3; +const MPL: usize = 3; + +fuzz_target!(|data: &[u8]| { + smol::block_on(async { + encoding_random::>(data).await; + }); +}); diff --git a/fuzz/fuzz_targets/path_encoding_random3.rs b/fuzz/fuzz_targets/path_encoding_random3.rs new file mode 100644 index 0000000..cc8f387 --- /dev/null +++ b/fuzz/fuzz_targets/path_encoding_random3.rs @@ -0,0 +1,15 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::encoding_random; + +const MCL: usize = 4; +const MCC: usize = 4; +const MPL: usize = 16; + +fuzz_target!(|data: &[u8]| { + smol::block_on(async { + encoding_random::>(data).await; + }); +}); diff --git a/fuzz/fuzz_targets/path_rel_path_encoding.rs b/fuzz/fuzz_targets/path_rel_path_encoding.rs index 7eb4390..f4db889 100644 --- a/fuzz/fuzz_targets/path_rel_path_encoding.rs +++ b/fuzz/fuzz_targets/path_rel_path_encoding.rs @@ -6,9 +6,9 @@ use ufotofu::local_nb::consumer::TestConsumer; use willow_data_model::path::Path; use willow_data_model_fuzz::encode::relative_encoding_roundtrip; -const MCL: usize = 16; -const MCC: usize = 16; -const MPL: usize = 16; +const MCL: usize = 300; +const MCC: usize = 300; +const MPL: usize = 300; fuzz_target!(|data: ( Path, diff --git a/fuzz/fuzz_targets/path_rel_path_encoding2.rs b/fuzz/fuzz_targets/path_rel_path_encoding2.rs new file mode 100644 index 0000000..a8fb0ed --- /dev/null +++ b/fuzz/fuzz_targets/path_rel_path_encoding2.rs @@ -0,0 +1,28 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; + +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; + +const MCL: usize = 2; +const MCC: usize = 3; +const MPL: usize = 3; + +fuzz_target!(|data: ( + Path, + Path, + TestConsumer +)| { + let (path_sub, path_ref, mut consumer) = data; + + smol::block_on(async { + relative_encoding_roundtrip::< + Path, + Path, + TestConsumer, + >(path_sub, path_ref, &mut consumer) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/path_rel_path_encoding3.rs b/fuzz/fuzz_targets/path_rel_path_encoding3.rs new file mode 100644 index 0000000..9f53c45 --- /dev/null +++ b/fuzz/fuzz_targets/path_rel_path_encoding3.rs @@ -0,0 +1,28 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ufotofu::local_nb::consumer::TestConsumer; + +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::relative_encoding_roundtrip; + +const MCL: usize = 4; +const MCC: usize = 4; +const MPL: usize = 16; + +fuzz_target!(|data: ( + Path, + Path, + TestConsumer +)| { + let (path_sub, path_ref, mut consumer) = data; + + smol::block_on(async { + relative_encoding_roundtrip::< + Path, + Path, + TestConsumer, + >(path_sub, path_ref, &mut consumer) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/path_rel_path_encoding_random.rs b/fuzz/fuzz_targets/path_rel_path_encoding_random.rs index 31186a6..cff2c7d 100644 --- a/fuzz/fuzz_targets/path_rel_path_encoding_random.rs +++ b/fuzz/fuzz_targets/path_rel_path_encoding_random.rs @@ -4,9 +4,9 @@ use libfuzzer_sys::fuzz_target; use willow_data_model::path::Path; use willow_data_model_fuzz::encode::relative_encoding_random; -const MCL: usize = 16; -const MCC: usize = 16; -const MPL: usize = 16; +const MCL: usize = 300; +const MCC: usize = 300; +const MPL: usize = 300; fuzz_target!(|data: (&[u8], Path)| { let (random_bytes, ref_path) = data; diff --git a/fuzz/fuzz_targets/path_rel_path_encoding_random2.rs b/fuzz/fuzz_targets/path_rel_path_encoding_random2.rs new file mode 100644 index 0000000..1531fe8 --- /dev/null +++ b/fuzz/fuzz_targets/path_rel_path_encoding_random2.rs @@ -0,0 +1,21 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::relative_encoding_random; + +const MCL: usize = 2; +const MCC: usize = 3; +const MPL: usize = 3; + +fuzz_target!(|data: (&[u8], Path)| { + let (random_bytes, ref_path) = data; + + smol::block_on(async { + relative_encoding_random::, Path>( + ref_path, + random_bytes, + ) + .await; + }); +}); diff --git a/fuzz/fuzz_targets/path_rel_path_encoding_random3.rs b/fuzz/fuzz_targets/path_rel_path_encoding_random3.rs new file mode 100644 index 0000000..8423fee --- /dev/null +++ b/fuzz/fuzz_targets/path_rel_path_encoding_random3.rs @@ -0,0 +1,21 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use willow_data_model::path::Path; +use willow_data_model_fuzz::encode::relative_encoding_random; + +const MCL: usize = 4; +const MCC: usize = 4; +const MPL: usize = 16; + +fuzz_target!(|data: (&[u8], Path)| { + let (random_bytes, ref_path) = data; + + smol::block_on(async { + relative_encoding_random::, Path>( + ref_path, + random_bytes, + ) + .await; + }); +}); From 6a53246f9d4431db33fc25ab96131de35d75dd53 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 00:40:40 +0200 Subject: [PATCH 32/40] Run clippy --fix --- data-model/src/encoding/relativity.rs | 2 +- data-model/src/encoding/shared_buffers.rs | 10 +++++----- data-model/src/path.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index a641621..fd35b80 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -180,7 +180,7 @@ impl RelativeDecodable ScratchSpacePathDecoding { /// /// Memory must have been initialised with a prior call to set_component_accumulated_length for the same `i`. unsafe fn get_component_accumulated_length(&self, i: usize) -> usize { - return MaybeUninit::assume_init(self.component_accumulated_lengths[i]); + MaybeUninit::assume_init(self.component_accumulated_lengths[i]) } /// Return a slice of the accumulated component lengths up to but excluding the `i`-th component, encoded as native-endian u8s. @@ -83,7 +83,7 @@ impl ScratchSpacePathDecoding { self.get_component_accumulated_length(i - 1) }; let end = self.get_component_accumulated_length(i); - return &mut self.path_data[start..end]; + &mut self.path_data[start..end] } /// Return a mutable slice of the path_data up to but excluding the i-th component. @@ -93,7 +93,7 @@ impl ScratchSpacePathDecoding { /// Accumulated component lengths for `i - 1` must have been set (unless `i == 0`). pub unsafe fn path_data_until_as_mut(&mut self, i: usize) -> &mut [MaybeUninit] { let end = self.get_component_accumulated_length(i - 1); - return &mut self.path_data[0..end]; + &mut self.path_data[0..end] } /// Get the path data of the first `i` components. @@ -118,7 +118,7 @@ impl ScratchSpacePathDecoding { /// The first `i` accumulated component lengths must have been set, and all corresponding path data must be initialised. MCL, MCC, and MCP are trusted blindly and must be adhered to by the data in the scratch buffer. pub unsafe fn to_path(&self, i: usize) -> Path { if i == 0 { - return Path::new_empty(); + Path::new_empty() } else { let total_length = if i == 0 { 0 @@ -131,7 +131,7 @@ impl ScratchSpacePathDecoding { buf.extend_from_slice(self.get_accumumulated_component_lengths(i)); buf.extend_from_slice(self.get_path_data(i)); - return Path::from_buffer_and_component_count(buf.freeze(), i); + Path::from_buffer_and_component_count(buf.freeze(), i) } } } diff --git a/data-model/src/path.rs b/data-model/src/path.rs index 3bbe3da..e241fb0 100644 --- a/data-model/src/path.rs +++ b/data-model/src/path.rs @@ -843,7 +843,7 @@ impl Decodable for Path Date: Wed, 24 Jul 2024 00:40:56 +0200 Subject: [PATCH 33/40] Run cargo fmt --- data-model/src/encoding/shared_buffers.rs | 15 +++++---------- fuzz/fuzz_targets/path.rs | 8 +++++--- fuzz/fuzz_targets/path2.rs | 8 +++++--- fuzz/fuzz_targets/path3.rs | 8 +++++--- fuzz/fuzz_targets/path4.rs | 8 +++++--- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/data-model/src/encoding/shared_buffers.rs b/data-model/src/encoding/shared_buffers.rs index 7863946..558cb31 100644 --- a/data-model/src/encoding/shared_buffers.rs +++ b/data-model/src/encoding/shared_buffers.rs @@ -35,20 +35,15 @@ impl ScratchSpacePathDecoding { } /// # Saftey - /// + /// /// UB if length of slice is greater than `size_of::() * MCC`. - pub unsafe fn set_many_component_accumulated_lengths_from_ne( - &mut self, - lengths: &[u8], - ) { + pub unsafe fn set_many_component_accumulated_lengths_from_ne(&mut self, lengths: &[u8]) { let slice: &mut [MaybeUninit] = core::slice::from_raw_parts_mut( - self.component_accumulated_lengths[..lengths.len() / size_of::()].as_mut_ptr() as *mut MaybeUninit, + self.component_accumulated_lengths[..lengths.len() / size_of::()].as_mut_ptr() + as *mut MaybeUninit, lengths.len(), ); - MaybeUninit::copy_from_slice( - slice, - lengths - ); + MaybeUninit::copy_from_slice(slice, lengths); } /// # Safety diff --git a/fuzz/fuzz_targets/path.rs b/fuzz/fuzz_targets/path.rs index 83469e5..0fc1ca9 100644 --- a/fuzz/fuzz_targets/path.rs +++ b/fuzz/fuzz_targets/path.rs @@ -14,15 +14,17 @@ fuzz_target!(|data: (CreatePath, CreatePath)| { let p2 = create_path::<300, 300, 300>(&cp2); match (ctrl1, p1) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl1), Ok(p1)) => { match (ctrl2, p2) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl2), Ok(p2)) => { assert_isomorphic_paths(&ctrl1, &ctrl2, &p1, &p2); } _ => { - panic!("Create_path_rc and create_path must either both succeeed or both fail."); + panic!( + "Create_path_rc and create_path must either both succeeed or both fail." + ); } } } diff --git a/fuzz/fuzz_targets/path2.rs b/fuzz/fuzz_targets/path2.rs index 7ceacd4..80cb63a 100644 --- a/fuzz/fuzz_targets/path2.rs +++ b/fuzz/fuzz_targets/path2.rs @@ -14,15 +14,17 @@ fuzz_target!(|data: (CreatePath, CreatePath)| { let p2 = create_path::<2, 3, 3>(&cp2); match (ctrl1, p1) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl1), Ok(p1)) => { match (ctrl2, p2) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl2), Ok(p2)) => { assert_isomorphic_paths(&ctrl1, &ctrl2, &p1, &p2); } _ => { - panic!("Create_path_rc and create_path must either both succeeed or both fail."); + panic!( + "Create_path_rc and create_path must either both succeeed or both fail." + ); } } } diff --git a/fuzz/fuzz_targets/path3.rs b/fuzz/fuzz_targets/path3.rs index 5cc9019..d5455f9 100644 --- a/fuzz/fuzz_targets/path3.rs +++ b/fuzz/fuzz_targets/path3.rs @@ -14,15 +14,17 @@ fuzz_target!(|data: (CreatePath, CreatePath)| { let p2 = create_path::<4, 4, 16>(&cp2); match (ctrl1, p1) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl1), Ok(p1)) => { match (ctrl2, p2) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl2), Ok(p2)) => { assert_isomorphic_paths(&ctrl1, &ctrl2, &p1, &p2); } _ => { - panic!("Create_path_rc and create_path must either both succeeed or both fail."); + panic!( + "Create_path_rc and create_path must either both succeeed or both fail." + ); } } } diff --git a/fuzz/fuzz_targets/path4.rs b/fuzz/fuzz_targets/path4.rs index e2658eb..fe4ab85 100644 --- a/fuzz/fuzz_targets/path4.rs +++ b/fuzz/fuzz_targets/path4.rs @@ -14,15 +14,17 @@ fuzz_target!(|data: (CreatePath, CreatePath)| { let p2 = create_path::<3, 3, 3>(&cp2); match (ctrl1, p1) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl1), Ok(p1)) => { match (ctrl2, p2) { - (Err(_), Err(_)) => { /* no-op */} + (Err(_), Err(_)) => { /* no-op */ } (Ok(ctrl2), Ok(p2)) => { assert_isomorphic_paths(&ctrl1, &ctrl2, &p1, &p2); } _ => { - panic!("Create_path_rc and create_path must either both succeeed or both fail."); + panic!( + "Create_path_rc and create_path must either both succeeed or both fail." + ); } } } From 629c69568e44675ed9acc69afd13c0a7bf3fad2c Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 24 Jul 2024 10:01:57 +0100 Subject: [PATCH 34/40] Fix typo --- data-model/src/encoding/shared_buffers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/encoding/shared_buffers.rs b/data-model/src/encoding/shared_buffers.rs index 558cb31..29d45fe 100644 --- a/data-model/src/encoding/shared_buffers.rs +++ b/data-model/src/encoding/shared_buffers.rs @@ -34,7 +34,7 @@ impl ScratchSpacePathDecoding { ); } - /// # Saftey + /// # Safety /// /// UB if length of slice is greater than `size_of::() * MCC`. pub unsafe fn set_many_component_accumulated_lengths_from_ne(&mut self, lengths: &[u8]) { From df6669650068641d2fc291ce019277a579bc11a2 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 12:06:54 +0200 Subject: [PATCH 35/40] Make fields private --- data-model/src/encoding/relativity.rs | 402 +++++++++++++------------- data-model/src/entry.rs | 61 +++- data-model/src/grouping/area.rs | 56 +++- data-model/src/grouping/range_3d.rs | 59 +++- 4 files changed, 356 insertions(+), 222 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index a7aee2e..aee0d57 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -20,7 +20,6 @@ use crate::{ path::Path, }; - use super::shared_buffers::ScratchSpacePathDecoding; /// A type that can be used to encode `T` to a bytestring *encoded relative to `R`*. @@ -207,43 +206,45 @@ where where Consumer: BulkConsumer, { - let time_diff = self.timestamp.abs_diff(reference.timestamp); + let time_diff = self.get_timestamp().abs_diff(reference.get_timestamp()); let mut header: u8 = 0b0000_0000; - if self.namespace_id != reference.namespace_id { + if self.get_namespace_id() != reference.get_namespace_id() { header |= 0b1000_0000; } - if self.subspace_id != reference.subspace_id { + if self.get_subspace_id() != reference.get_subspace_id() { header |= 0b0100_0000; } - if self.timestamp > reference.timestamp { + if self.get_timestamp() > reference.get_timestamp() { header |= 0b0010_0000; } header |= CompactWidth::from_u64(time_diff).bitmask(4); - header |= CompactWidth::from_u64(self.payload_length).bitmask(6); + header |= CompactWidth::from_u64(self.get_payload_length()).bitmask(6); consumer.consume(header).await?; - if self.namespace_id != reference.namespace_id { - self.namespace_id.encode(consumer).await?; + if self.get_namespace_id() != reference.get_namespace_id() { + self.get_namespace_id().encode(consumer).await?; } - if self.subspace_id != reference.subspace_id { - self.subspace_id.encode(consumer).await?; + if self.get_subspace_id() != reference.get_subspace_id() { + self.get_subspace_id().encode(consumer).await?; } - self.path.relative_encode(&reference.path, consumer).await?; + self.get_path() + .relative_encode(&reference.get_path(), consumer) + .await?; encode_compact_width_be(time_diff, consumer).await?; - encode_compact_width_be(self.payload_length, consumer).await?; + encode_compact_width_be(self.get_payload_length(), consumer).await?; - self.payload_digest.encode(consumer).await?; + self.get_payload_digest().encode(consumer).await?; Ok(()) } @@ -283,13 +284,13 @@ where let namespace_id = if is_namespace_encoded { N::decode(producer).await? } else { - reference.namespace_id.clone() + reference.get_namespace_id().clone() }; /* // Verify that the encoded namespace wasn't the same as ours // Which would indicate invalid input - if is_namespace_encoded && namespace_id == reference.namespace_id { + if is_namespace_encoded && namespace_id == reference.get_namespace_id() { return Err(DecodeError::InvalidInput); } */ @@ -297,37 +298,37 @@ where let subspace_id = if is_subspace_encoded { S::decode(producer).await? } else { - reference.subspace_id.clone() + reference.get_subspace_id().clone() }; /* // Verify that the encoded subspace wasn't the same as ours // Which would indicate invalid input - if is_subspace_encoded && subspace_id == reference.subspace_id { + if is_subspace_encoded && subspace_id == reference.get_subspace_id() { return Err(DecodeError::InvalidInput); } */ - let path = Path::::relative_decode(&reference.path, producer).await?; + let path = Path::::relative_decode(&reference.get_path(), producer).await?; let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; // Add or subtract safely here to avoid overflows caused by malicious or faulty encodings. let timestamp = if add_or_subtract_time_diff { reference - .timestamp + .get_timestamp() .checked_add(time_diff) .ok_or(DecodeError::InvalidInput)? } else { reference - .timestamp + .get_timestamp() .checked_sub(time_diff) .ok_or(DecodeError::InvalidInput)? }; /* // Verify that the correct add_or_subtract_time_diff flag was set. - let should_have_subtracted = timestamp <= reference.timestamp; + let should_have_subtracted = timestamp <= reference.get_timestamp(); if add_or_subtract_time_diff && should_have_subtracted { return Err(DecodeError::InvalidInput); } @@ -338,14 +339,14 @@ where let payload_digest = PD::decode(producer).await?; - Ok(Entry { + Ok(Entry::new( namespace_id, subspace_id, path, timestamp, payload_length, payload_digest, - }) + )) } } @@ -369,7 +370,7 @@ where { let (namespace, out) = reference; - if &self.namespace_id != namespace { + if self.get_namespace_id() != namespace { panic!("Tried to encode an entry relative to a namespace it does not belong to") } @@ -378,33 +379,37 @@ where } let time_diff = core::cmp::min( - self.timestamp - out.times.start, - u64::from(&out.times.end) - self.timestamp, + self.get_timestamp() - out.get_times().start, + u64::from(&out.get_times().end) - self.get_timestamp(), ); let mut header = 0b0000_0000; - if out.subspace == AreaSubspace::Any { + if out.get_subspace().is_any() { header |= 0b1000_0000; } - if self.timestamp - out.times.start <= u64::from(&out.times.end) - self.timestamp { + if self.get_timestamp() - out.get_times().start + <= u64::from(&out.get_times().end) - self.get_timestamp() + { header |= 0b0100_0000; } header |= CompactWidth::from_u64(time_diff).bitmask(2); - header |= CompactWidth::from_u64(self.payload_length).bitmask(4); + header |= CompactWidth::from_u64(self.get_payload_length()).bitmask(4); consumer.consume(header).await?; - if out.subspace == AreaSubspace::Any { - self.subspace_id.encode(consumer).await?; + if out.get_subspace().is_any() { + self.get_subspace_id().encode(consumer).await?; } - self.path.relative_encode(&out.path, consumer).await?; + self.get_path() + .relative_encode(&out.get_path(), consumer) + .await?; encode_compact_width_be(time_diff, consumer).await?; - encode_compact_width_be(self.payload_length, consumer).await?; - self.payload_digest.encode(consumer).await?; + encode_compact_width_be(self.get_payload_length(), consumer).await?; + self.get_payload_digest().encode(consumer).await?; Ok(()) } @@ -442,20 +447,20 @@ where } let subspace_id = if is_subspace_encoded { - match &out.subspace { + match &out.get_subspace() { AreaSubspace::Any => S::decode(producer).await?, AreaSubspace::Id(_) => return Err(DecodeError::InvalidInput), } } else { - match &out.subspace { + match &out.get_subspace() { AreaSubspace::Any => return Err(DecodeError::InvalidInput), AreaSubspace::Id(id) => id.clone(), } }; - let path = Path::relative_decode(&out.path, producer).await?; + let path = Path::relative_decode(&out.get_path(), producer).await?; - if !path.is_prefixed_by(&out.path) { + if !path.is_prefixed_by(&out.get_path()) { return Err(DecodeError::InvalidInput); } @@ -466,34 +471,34 @@ where let payload_digest = PD::decode(producer).await?; let timestamp = if add_time_diff_to_start { - out.times.start.checked_add(time_diff) + out.get_times().start.checked_add(time_diff) } else { - u64::from(&out.times.end).checked_sub(time_diff) + u64::from(&out.get_times().end).checked_sub(time_diff) } .ok_or(DecodeError::InvalidInput)?; // === Necessary to produce canonic encodings. === // Verify that the correct add_or_subtract_time_diff flag was set. - let should_have_added = timestamp.checked_sub(out.times.start) - <= u64::from(&out.times.end).checked_sub(timestamp); + let should_have_added = timestamp.checked_sub(out.get_times().start) + <= u64::from(&out.get_times().end).checked_sub(timestamp); if add_time_diff_to_start != should_have_added { return Err(DecodeError::InvalidInput); } // =============================================== - if !out.times.includes(×tamp) { + if !out.get_times().includes(×tamp) { return Err(DecodeError::InvalidInput); } - Ok(Self { - namespace_id: namespace.clone(), + Ok(Self::new( + namespace.clone(), subspace_id, path, timestamp, payload_length, payload_digest, - }) + )) } } @@ -517,7 +522,7 @@ where { let (namespace, out) = reference; - if &self.namespace_id != namespace { + if self.get_namespace_id() != namespace { panic!("Tried to encode an entry relative to a namespace it does not belong to") } @@ -526,22 +531,25 @@ where } let time_diff = core::cmp::min( - self.timestamp.abs_diff(out.times.start), - self.timestamp.abs_diff(u64::from(&out.times.end)), + self.get_timestamp().abs_diff(out.get_times().start), + self.get_timestamp() + .abs_diff(u64::from(&out.get_times().end)), ); let mut header = 0b0000_0000; - // Encode e.subspace_id? - if self.subspace_id != out.subspaces.start { + // Encode e.get_subspace_id()? + if self.get_subspace_id() != &out.get_subspaces().start { header |= 0b1000_0000; } - // Encode e.path relative to out.paths.start or to out.paths.end? - let encode_path_relative_to_start = match &out.paths.end { + // Encode e.get_path() relative to out.get_paths().start or to out.get_paths().end? + let encode_path_relative_to_start = match &out.get_paths().end { RangeEnd::Closed(end_path) => { - let start_lcp = self.path.longest_common_prefix(&out.paths.start); - let end_lcp = self.path.longest_common_prefix(end_path); + let start_lcp = self + .get_path() + .longest_common_prefix(&out.get_paths().start); + let end_lcp = self.get_path().longest_common_prefix(end_path); start_lcp.get_component_count() >= end_lcp.get_component_count() } @@ -552,8 +560,9 @@ where header |= 0b0100_0000; } - // Add time_diff to out.times.start, or subtract from out.times.end? - let add_time_diff_with_start = time_diff == self.timestamp.abs_diff(out.times.start); + // Add time_diff to out.get_times().start, or subtract from out.get_times().end? + let add_time_diff_with_start = + time_diff == self.get_timestamp().abs_diff(out.get_times().start); if add_time_diff_with_start { header |= 0b0010_0000; @@ -562,36 +571,36 @@ where // 2-bit integer n such that 2^n gives compact_width(time_diff) header |= CompactWidth::from_u64(time_diff).bitmask(4); - // 2-bit integer n such that 2^n gives compact_width(e.payload_length) - header |= CompactWidth::from_u64(self.payload_length).bitmask(6); + // 2-bit integer n such that 2^n gives compact_width(e.get_payload_length()) + header |= CompactWidth::from_u64(self.get_payload_length()).bitmask(6); consumer.consume(header).await?; - if self.subspace_id != out.subspaces.start { - self.subspace_id.encode(consumer).await?; + if self.get_subspace_id() != &out.get_subspaces().start { + self.get_subspace_id().encode(consumer).await?; } - // Encode e.path relative to out.paths.start or to out.paths.end? - match &out.paths.end { + // Encode e.get_path() relative to out.get_paths().start or to out.get_paths().end? + match &out.get_paths().end { RangeEnd::Closed(end_path) => { if encode_path_relative_to_start { - self.path - .relative_encode(&out.paths.start, consumer) + self.get_path() + .relative_encode(&out.get_paths().start, consumer) .await?; } else { - self.path.relative_encode(end_path, consumer).await?; + self.get_path().relative_encode(end_path, consumer).await?; } } RangeEnd::Open => { - self.path - .relative_encode(&out.paths.start, consumer) + self.get_path() + .relative_encode(&out.get_paths().start, consumer) .await?; } } encode_compact_width_be(time_diff, consumer).await?; - encode_compact_width_be(self.payload_length, consumer).await?; - self.payload_digest.encode(consumer).await?; + encode_compact_width_be(self.get_payload_length(), consumer).await?; + self.get_payload_digest().encode(consumer).await?; Ok(()) } @@ -619,13 +628,13 @@ where let header = produce_byte(producer).await?; - // Decode e.subspace_id? + // Decode e.get_subspace_id()? let is_subspace_encoded = is_bitflagged(header, 0); - // Decode e.path relative to out.paths.start or to out.paths.end? + // Decode e.get_path() relative to out.get_paths().start or to out.get_paths().end? let decode_path_relative_to_start = is_bitflagged(header, 1); - // Add time_diff to out.times.start, or subtract from out.times.end? + // Add time_diff to out.get_times().start, or subtract from out.get_times().end? let add_time_diff_with_start = is_bitflagged(header, 2); if is_bitflagged(header, 3) { @@ -638,40 +647,40 @@ where let subspace_id = if is_subspace_encoded { S::decode(producer).await? } else { - out.subspaces.start.clone() + out.get_subspaces().start.clone() }; // === Necessary to produce canonic encodings. === // Verify that encoding the subspace was necessary. - if subspace_id == out.subspaces.start && is_subspace_encoded { + if subspace_id == out.get_subspaces().start && is_subspace_encoded { return Err(DecodeError::InvalidInput); } // =============================================== // Verify that subspace is included by range - if !out.subspaces.includes(&subspace_id) { + if !out.get_subspaces().includes(&subspace_id) { return Err(DecodeError::InvalidInput); } let path = if decode_path_relative_to_start { - Path::relative_decode(&out.paths.start, producer).await? + Path::relative_decode(&out.get_paths().start, producer).await? } else { - match &out.paths.end { + match &out.get_paths().end { RangeEnd::Closed(end_path) => Path::relative_decode(end_path, producer).await?, RangeEnd::Open => return Err(DecodeError::InvalidInput), } }; // Verify that path is included by range - if !out.paths.includes(&path) { + if !out.get_paths().includes(&path) { return Err(DecodeError::InvalidInput); } // === Necessary to produce canonic encodings. === // Verify that the path was encoded relative to the correct bound of the referenc path range. - let should_have_encoded_path_relative_to_start = match &out.paths.end { + let should_have_encoded_path_relative_to_start = match &out.get_paths().end { RangeEnd::Closed(end_path) => { - let start_lcp = path.longest_common_prefix(&out.paths.start); + let start_lcp = path.longest_common_prefix(&out.get_paths().start); let end_lcp = path.longest_common_prefix(end_path); start_lcp.get_component_count() >= end_lcp.get_component_count() @@ -692,25 +701,25 @@ where let payload_digest = PD::decode(producer).await?; let timestamp = if add_time_diff_with_start { - out.times.start.checked_add(time_diff) + out.get_times().start.checked_add(time_diff) } else { - match &out.times.end { + match &out.get_times().end { RangeEnd::Closed(end_time) => end_time.checked_sub(time_diff), - RangeEnd::Open => u64::from(&out.times.end).checked_sub(time_diff), + RangeEnd::Open => u64::from(&out.get_times().end).checked_sub(time_diff), } } .ok_or(DecodeError::InvalidInput)?; // Verify that timestamp is included by range - if !out.times.includes(×tamp) { + if !out.get_times().includes(×tamp) { return Err(DecodeError::InvalidInput); } // === Necessary to produce canonic encodings. === // Verify that time_diff is what it should have been let correct_time_diff = core::cmp::min( - timestamp.abs_diff(out.times.start), - timestamp.abs_diff(u64::from(&out.times.end)), + timestamp.abs_diff(out.get_times().start), + timestamp.abs_diff(u64::from(&out.get_times().end)), ); if time_diff != correct_time_diff { @@ -718,21 +727,21 @@ where } // Verify that the combine with start bitflag in the header was correct - let should_have_added_to_start = time_diff == timestamp.abs_diff(out.times.start); + let should_have_added_to_start = time_diff == timestamp.abs_diff(out.get_times().start); if should_have_added_to_start != add_time_diff_with_start { return Err(DecodeError::InvalidInput); } // ============================================== - Ok(Self { - namespace_id: namespace.clone(), + Ok(Self::new( + namespace.clone(), subspace_id, path, timestamp, payload_length, payload_digest, - }) + )) } } @@ -757,31 +766,31 @@ where } let start_diff = core::cmp::min( - self.times.start - out.times.start, - u64::from(&out.times.end) - self.times.start, + self.get_times().start - out.get_times().start, + u64::from(&out.get_times().end) - self.get_times().start, ); let end_diff = core::cmp::min( - u64::from(&self.times.end) - out.times.start, - u64::from(&out.times.end) - u64::from(&self.times.end), + u64::from(&self.get_times().end) - out.get_times().start, + u64::from(&out.get_times().end) - u64::from(&self.get_times().end), ); let mut header = 0; - if self.subspace != out.subspace { + if self.get_subspace() != out.get_subspace() { header |= 0b1000_0000; } - if self.times.end == RangeEnd::Open { + if self.get_times().end == RangeEnd::Open { header |= 0b0100_0000; } - if start_diff == self.times.start - out.times.start { + if start_diff == self.get_times().start - out.get_times().start { header |= 0b0010_0000; } - if self.times.end != RangeEnd::Open - && end_diff == u64::from(&self.times.end) - out.times.start + if self.get_times().end != RangeEnd::Open + && end_diff == u64::from(&self.get_times().end) - out.get_times().start { header |= 0b0001_0000; } @@ -791,7 +800,7 @@ where consumer.consume(header).await?; - match (&self.subspace, &out.subspace) { + match (&self.get_subspace(), &out.get_subspace()) { (AreaSubspace::Any, AreaSubspace::Any) => {} // Same subspace (AreaSubspace::Id(_), AreaSubspace::Id(_)) => {} // Same subspace (AreaSubspace::Id(subspace), AreaSubspace::Any) => { @@ -804,11 +813,13 @@ where } } - self.path.relative_encode(&out.path, consumer).await?; + self.get_path() + .relative_encode(&out.get_path(), consumer) + .await?; encode_compact_width_be(start_diff, consumer).await?; - if self.times.end != RangeEnd::Open { + if self.get_times().end != RangeEnd::Open { encode_compact_width_be(end_diff, consumer).await?; } @@ -840,10 +851,10 @@ where // Decode end value of times? let is_times_end_open = is_bitflagged(header, 1); - // Add start_diff to out.times.start, or subtract from out.times.end? + // Add start_diff to out.get_times().start, or subtract from out.get_times().end? let add_start_diff = is_bitflagged(header, 2); - // Add end_diff to out.times.start, or subtract from out.times.end? + // Add end_diff to out.get_times().start, or subtract from out.get_times().end? let add_end_diff = is_bitflagged(header, 3); // === Necessary to produce canonic encodings. === @@ -869,18 +880,18 @@ where // === Necessary to produce canonic encodings. === // Verify that subspace wasn't needlessly encoded - if sub == out.subspace { + if &sub == out.get_subspace() { return Err(DecodeError::InvalidInput); } // =============================================== sub } else { - out.subspace.clone() + out.get_subspace().clone() }; // Verify that the decoded subspace is included by the reference subspace - match (&out.subspace, &subspace) { + match (&out.get_subspace(), &subspace) { (AreaSubspace::Any, AreaSubspace::Any) => {} (AreaSubspace::Any, AreaSubspace::Id(_)) => {} (AreaSubspace::Id(_), AreaSubspace::Any) => { @@ -893,26 +904,26 @@ where } } - let path = Path::relative_decode(&out.path, producer).await?; + let path = Path::relative_decode(&out.get_path(), producer).await?; // Verify the decoded path is prefixed by the reference path - if !path.is_prefixed_by(&out.path) { + if !path.is_prefixed_by(&out.get_path()) { return Err(DecodeError::InvalidInput); } let start_diff = decode_compact_width_be(start_diff_compact_width, producer).await?; let start = if add_start_diff { - out.times.start.checked_add(start_diff) + out.get_times().start.checked_add(start_diff) } else { - u64::from(&out.times.end).checked_sub(start_diff) + u64::from(&out.get_times().end).checked_sub(start_diff) } .ok_or(DecodeError::InvalidInput)?; // Verify they sent correct start diff let expected_start_diff = core::cmp::min( - start.checked_sub(out.times.start), - u64::from(&out.times.end).checked_sub(start), + start.checked_sub(out.get_times().start), + u64::from(&out.get_times().end).checked_sub(start), ) .ok_or(DecodeError::InvalidInput)?; @@ -924,7 +935,7 @@ where // Verify that bit 2 of the header was set correctly let should_add_start_diff = start_diff == start - .checked_sub(out.times.start) + .checked_sub(out.get_times().start) .ok_or(DecodeError::InvalidInput)?; if add_start_diff != should_add_start_diff { @@ -942,16 +953,16 @@ where let end_diff = decode_compact_width_be(end_diff_compact_width, producer).await?; let end = if add_end_diff { - out.times.start.checked_add(end_diff) + out.get_times().start.checked_add(end_diff) } else { - u64::from(&out.times.end).checked_sub(end_diff) + u64::from(&out.get_times().end).checked_sub(end_diff) } .ok_or(DecodeError::InvalidInput)?; // Verify they sent correct end diff let expected_end_diff = core::cmp::min( - end.checked_sub(out.times.start), - u64::from(&out.times.end).checked_sub(end), + end.checked_sub(out.get_times().start), + u64::from(&out.get_times().end).checked_sub(end), ) .ok_or(DecodeError::InvalidInput)?; @@ -962,7 +973,7 @@ where // === Necessary to produce canonic encodings. === let should_add_end_diff = end_diff == end - .checked_sub(out.times.start) + .checked_sub(out.get_times().start) .ok_or(DecodeError::InvalidInput)?; if add_end_diff != should_add_end_diff { @@ -976,15 +987,11 @@ where let times = Range { start, end }; // Verify the decoded time range is included by the reference time range - if !out.times.includes_range(×) { + if !out.get_times().includes_range(×) { return Err(DecodeError::InvalidInput); } - Ok(Self { - subspace, - path, - times, - }) + Ok(Self::new(subspace, path, times)) } } @@ -1004,16 +1011,16 @@ where where Consumer: BulkConsumer, { - let start_to_start = self.times.start.abs_diff(reference.times.start); - let start_to_end = match reference.times.end { - RangeEnd::Closed(end) => self.times.start.abs_diff(end), + let start_to_start = self.get_times().start.abs_diff(reference.get_times().start); + let start_to_end = match reference.get_times().end { + RangeEnd::Closed(end) => self.get_times().start.abs_diff(end), RangeEnd::Open => u64::MAX, }; - let end_to_start = match self.times.end { - RangeEnd::Closed(end) => end.abs_diff(reference.times.start), + let end_to_start = match self.get_times().end { + RangeEnd::Closed(end) => end.abs_diff(reference.get_times().start), RangeEnd::Open => u64::MAX, }; - let end_to_end = match (&self.times.end, &reference.times.end) { + let end_to_end = match (&self.get_times().end, &reference.get_times().end) { (RangeEnd::Closed(self_end), RangeEnd::Closed(ref_end)) => self_end.abs_diff(*ref_end), (RangeEnd::Closed(_), RangeEnd::Open) => u64::MAX, (RangeEnd::Open, RangeEnd::Closed(_)) => u64::MAX, @@ -1026,33 +1033,33 @@ where let mut header_1 = 0b0000_0000; - // Bits 0, 1 - Encode r.subspaces.start? - if self.subspaces.start == reference.subspaces.start { + // Bits 0, 1 - Encode r.get_subspaces().start? + if self.get_subspaces().start == reference.get_subspaces().start { header_1 |= 0b0100_0000; - } else if reference.subspaces.end == self.subspaces.start { + } else if reference.get_subspaces().end == self.get_subspaces().start { header_1 |= 0b1000_0000; } else { header_1 |= 0b1100_0000; } - // Bits 2, 3 - Encode r.subspaces.end? - if self.subspaces.end == RangeEnd::Open { + // Bits 2, 3 - Encode r.get_subspaces().end? + if self.get_subspaces().end == RangeEnd::Open { // Do nothing - } else if self.subspaces.end == reference.subspaces.start { + } else if self.get_subspaces().end == reference.get_subspaces().start { header_1 |= 0b0001_0000; - } else if self.subspaces.end == reference.subspaces.end { + } else if self.get_subspaces().end == reference.get_subspaces().end { header_1 |= 0b0010_0000; - } else if self.subspaces.end != RangeEnd::Open { + } else if self.get_subspaces().end != RangeEnd::Open { header_1 |= 0b0011_0000; } - // Bit 4 - Encode r.paths.start relative to ref.paths.start or to ref.paths.end? - if let RangeEnd::Closed(ref_path_end) = &reference.paths.end { + // Bit 4 - Encode r.get_paths().start relative to ref.get_paths().start or to ref.get_paths().end? + if let RangeEnd::Closed(ref_path_end) = &reference.get_paths().end { let lcp_start_start = self - .paths + .get_paths() .start - .longest_common_prefix(&reference.paths.start); - let lcp_start_end = self.paths.start.longest_common_prefix(ref_path_end); + .longest_common_prefix(&reference.get_paths().start); + let lcp_start_end = self.get_paths().start.longest_common_prefix(ref_path_end); if lcp_start_start.get_component_count() >= lcp_start_end.get_component_count() { header_1 |= 0b0000_1000; @@ -1062,14 +1069,15 @@ where } // Bit 5 - Self path end open? - if self.paths.end == RangeEnd::Open { + if self.get_paths().end == RangeEnd::Open { header_1 |= 0b0000_0100; } - // Bit 6 - Encode r.paths.end relative to ref.paths.start or to ref.paths.end (if at all)? - match (&self.paths.end, &reference.paths.end) { + // Bit 6 - Encode r.get_paths().end relative to ref.get_paths().start or to ref.get_paths().end (if at all)? + match (&self.get_paths().end, &reference.get_paths().end) { (RangeEnd::Closed(self_path_end), RangeEnd::Closed(ref_path_end)) => { - let lcp_end_start = self_path_end.longest_common_prefix(&reference.paths.start); + let lcp_end_start = + self_path_end.longest_common_prefix(&reference.get_paths().start); let lcp_end_end = self_path_end.longest_common_prefix(ref_path_end); if lcp_end_start.get_component_count() > lcp_end_end.get_component_count() { @@ -1084,7 +1092,7 @@ where } // Bit 7 - Self time end open? - if self.times.end == RangeEnd::Open { + if self.get_times().end == RangeEnd::Open { header_1 |= 0b0000_0001; } @@ -1092,14 +1100,14 @@ where let mut header_2 = 0b0000_0000; - // Bit 8 - Encode r.times.start relative to ref.times.start or ref.times.end? + // Bit 8 - Encode r.get_times().start relative to ref.get_times().start or ref.get_times().end? if start_to_start <= start_to_end { header_2 |= 0b1000_0000; } // Bit 9 -Add or subtract start_time_diff? - if is_bitflagged(header_2, 0) && self.times.start >= reference.times.start - || !is_bitflagged(header_2, 0) && self.times.start >= reference.times.end + if is_bitflagged(header_2, 0) && self.get_times().start >= reference.get_times().start + || !is_bitflagged(header_2, 0) && self.get_times().start >= reference.get_times().end { header_2 |= 0b0100_0000; } @@ -1107,22 +1115,23 @@ where // Bit 10, 11 - 2-bit integer n such that 2^n gives compact_width(start_time_diff) header_2 |= CompactWidth::from_u64(start_time_diff).bitmask(2); - // Bit 12 - Encode r.times.end relative to ref.times.start or ref.times.end (if at all)? - if self.times.end != RangeEnd::Open && end_to_start <= end_to_end { + // Bit 12 - Encode r.get_times().end relative to ref.get_times().start or ref.get_times().end (if at all)? + if self.get_times().end != RangeEnd::Open && end_to_start <= end_to_end { header_2 |= 0b0000_1000; } // Bit 13 - Add or subtract end_time_diff (if encoding it at all)? - if self.times.end == RangeEnd::Open { + if self.get_times().end == RangeEnd::Open { // do nothing - } else if (is_bitflagged(header_2, 4) && self.times.end >= reference.times.start) - || (!is_bitflagged(header_2, 4) && self.times.end >= reference.times.end) + } else if (is_bitflagged(header_2, 4) + && self.get_times().end >= reference.get_times().start) + || (!is_bitflagged(header_2, 4) && self.get_times().end >= reference.get_times().end) { header_2 |= 0b0000_0100; } // Bits 14, 15 - ignored, or 2-bit integer n such that 2^n gives compact_width(end_time_diff) - if self.times.end == RangeEnd::Open { + if self.get_times().end == RangeEnd::Open { // do nothing } else { header_2 |= CompactWidth::from_u64(end_time_diff).bitmask(6); @@ -1130,38 +1139,41 @@ where consumer.consume(header_2).await?; - if (self.subspaces.start == reference.subspaces.start) - || (reference.subspaces.end == self.subspaces.start) + if (self.get_subspaces().start == reference.get_subspaces().start) + || (reference.get_subspaces().end == self.get_subspaces().start) { // Don't encode } else { - self.subspaces.start.encode(consumer).await?; + self.get_subspaces().start.encode(consumer).await?; } - if self.subspaces.end == RangeEnd::Open - || (self.subspaces.end == reference.subspaces.start) - || (self.subspaces.end == reference.subspaces.end) + if self.get_subspaces().end == RangeEnd::Open + || (self.get_subspaces().end == reference.get_subspaces().start) + || (self.get_subspaces().end == reference.get_subspaces().end) { // Don't encode end subspace - } else if let RangeEnd::Closed(end_subspace) = &self.subspaces.end { + } else if let RangeEnd::Closed(end_subspace) = &self.get_subspaces().end { end_subspace.encode(consumer).await?; } if is_bitflagged(header_1, 4) { - self.paths + self.get_paths() + .start + .relative_encode(&reference.get_paths().start, consumer) + .await?; + } else if let RangeEnd::Closed(end_path) = &reference.get_paths().end { + self.get_paths() .start - .relative_encode(&reference.paths.start, consumer) + .relative_encode(end_path, consumer) .await?; - } else if let RangeEnd::Closed(end_path) = &reference.paths.end { - self.paths.start.relative_encode(end_path, consumer).await?; } - if let RangeEnd::Closed(end_path) = &self.paths.end { + if let RangeEnd::Closed(end_path) = &self.get_paths().end { if is_bitflagged(header_1, 6) { end_path - .relative_encode(&reference.paths.start, consumer) + .relative_encode(&reference.get_paths().start, consumer) .await? - } else if let RangeEnd::Closed(ref_end_path) = &reference.paths.end { + } else if let RangeEnd::Closed(ref_end_path) = &reference.get_paths().end { end_path.relative_encode(ref_end_path, consumer).await?; } } @@ -1209,8 +1221,8 @@ where // Decode subspace start let subspace_start = match subspace_start_flags { - 0b0100_0000 => reference.subspaces.start.clone(), - 0b1000_0000 => match &reference.subspaces.end { + 0b0100_0000 => reference.get_subspaces().start.clone(), + 0b1000_0000 => match &reference.get_subspaces().end { RangeEnd::Closed(end) => end.clone(), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, @@ -1220,8 +1232,8 @@ where let subspace_end = match subspace_end_flags { 0b0000_0000 => RangeEnd::Open, - 0b0001_0000 => RangeEnd::Closed(reference.subspaces.start.clone()), - 0b0010_0000 => match &reference.subspaces.end { + 0b0001_0000 => RangeEnd::Closed(reference.get_subspaces().start.clone()), + 0b0010_0000 => match &reference.get_subspaces().end { RangeEnd::Closed(end) => RangeEnd::Closed(end.clone()), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, @@ -1229,12 +1241,12 @@ where _ => RangeEnd::Closed(S::decode(producer).await?), }; - let path_start = match (is_path_start_rel_to_start, &reference.paths.end) { + let path_start = match (is_path_start_rel_to_start, &reference.get_paths().end) { (true, RangeEnd::Closed(_)) => { - Path::relative_decode(&reference.paths.start, producer).await? + Path::relative_decode(&reference.get_paths().start, producer).await? } (true, RangeEnd::Open) => { - Path::relative_decode(&reference.paths.start, producer).await? + Path::relative_decode(&reference.get_paths().start, producer).await? } (false, RangeEnd::Closed(path_end)) => { Path::relative_decode(path_end, producer).await? @@ -1245,9 +1257,9 @@ where let path_end = if is_path_end_open { RangeEnd::Open } else if is_path_end_rel_to_start { - RangeEnd::Closed(Path::relative_decode(&reference.paths.start, producer).await?) + RangeEnd::Closed(Path::relative_decode(&reference.get_paths().start, producer).await?) } else { - match &reference.paths.end { + match &reference.get_paths().end { RangeEnd::Closed(end) => { RangeEnd::Closed(Path::relative_decode(end, producer).await?) } @@ -1259,13 +1271,13 @@ where decode_compact_width_be(start_time_diff_compact_width, producer).await?; let time_start = match (is_time_start_rel_to_start, add_or_subtract_start_time_diff) { - (true, true) => reference.times.start.checked_add(start_time_diff), - (true, false) => reference.times.start.checked_sub(start_time_diff), - (false, true) => match reference.times.end { + (true, true) => reference.get_times().start.checked_add(start_time_diff), + (true, false) => reference.get_times().start.checked_sub(start_time_diff), + (false, true) => match reference.get_times().end { RangeEnd::Closed(ref_end) => ref_end.checked_add(start_time_diff), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, - (false, false) => match reference.times.end { + (false, false) => match reference.get_times().end { RangeEnd::Closed(ref_end) => ref_end.checked_sub(start_time_diff), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, @@ -1280,19 +1292,19 @@ where match (is_times_end_rel_to_start, add_or_subtract_end_time_diff) { (true, true) => RangeEnd::Closed( reference - .times + .get_times() .start .checked_add(end_time_diff) .ok_or(DecodeError::InvalidInput)?, ), (true, false) => RangeEnd::Closed( reference - .times + .get_times() .start .checked_sub(end_time_diff) .ok_or(DecodeError::InvalidInput)?, ), - (false, true) => match reference.times.end { + (false, true) => match reference.get_times().end { RangeEnd::Closed(ref_end) => RangeEnd::Closed( ref_end .checked_add(end_time_diff) @@ -1300,7 +1312,7 @@ where ), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, - (false, false) => match reference.times.end { + (false, false) => match reference.get_times().end { RangeEnd::Closed(ref_end) => RangeEnd::Closed( ref_end .checked_sub(end_time_diff) @@ -1311,19 +1323,19 @@ where } }; - Ok(Self { - subspaces: Range { + Ok(Self::new( + Range { start: subspace_start, end: subspace_end, }, - paths: Range { + Range { start: path_start, end: path_end, }, - times: Range { + Range { start: time_start, end: time_end, }, - }) + )) } } diff --git a/data-model/src/entry.rs b/data-model/src/entry.rs index a58b3a7..7933e53 100644 --- a/data-model/src/entry.rs +++ b/data-model/src/entry.rs @@ -36,17 +36,17 @@ where PD: PayloadDigest, { /// The identifier of the namespace to which the [`Entry`] belongs. - pub namespace_id: N, + namespace_id: N, /// The identifier of the subspace to which the [`Entry`] belongs. - pub subspace_id: S, + subspace_id: S, /// The [`Path`] to which the [`Entry`] was written. - pub path: Path, + path: Path, /// The claimed creation time of the [`Entry`]. - pub timestamp: Timestamp, + timestamp: Timestamp, /// The length of the Payload in bytes. - pub payload_length: u64, + payload_length: u64, /// The result of applying hash_payload to the Payload. - pub payload_digest: PD, + payload_digest: PD, } impl Entry @@ -55,6 +55,55 @@ where S: SubspaceId, PD: PayloadDigest, { + /// Create a new [`Entry`]. + pub fn new( + namespace_id: N, + subspace_id: S, + path: Path, + timestamp: Timestamp, + payload_length: u64, + payload_digest: PD, + ) -> Self { + Entry { + namespace_id, + subspace_id, + path, + timestamp, + payload_length, + payload_digest, + } + } + + /// Return a reference to the identifier of the namespace to which the [`Entry`] belongs. + pub fn get_namespace_id(&self) -> &N { + &self.namespace_id + } + + /// Return a reference to the identifier of the subspace_id to which the [`Entry`] belongs. + pub fn get_subspace_id(&self) -> &S { + &self.subspace_id + } + + /// Return a reference to the [`Path`] to which the [`Entry`] was written. + pub fn get_path(&self) -> &Path { + &self.path + } + + /// Return the claimed creation time of the [`Entry`]. + pub fn get_timestamp(&self) -> Timestamp { + self.timestamp + } + + /// Return the length of the Payload in bytes. + pub fn get_payload_length(&self) -> u64 { + self.payload_length + } + + /// Return a reference to the result of applying hash_payload to the Payload. + pub fn get_payload_digest(&self) -> &PD { + &self.payload_digest + } + /// Return if this [`Entry`] is newer than another using their timestamps. /// Tie-breaks using the Entries' payload digest and payload length otherwise. /// [Definition](https://willowprotocol.org/specs/data-model/index.html#entry_newer). diff --git a/data-model/src/grouping/area.rs b/data-model/src/grouping/area.rs index 10a274f..11849dd 100644 --- a/data-model/src/grouping/area.rs +++ b/data-model/src/grouping/area.rs @@ -18,7 +18,7 @@ pub enum AreaSubspace { } impl AreaSubspace { - /// Whether this [`AreaSubspace`] includes a given [`SubspaceId`]. + /// Return whether this [`AreaSubspace`] includes a given [`SubspaceId`]. pub fn includes(&self, sub: &S) -> bool { match self { AreaSubspace::Any => true, @@ -36,6 +36,14 @@ impl AreaSubspace { (Self::Id(_a), Self::Id(_b)) => None, } } + + /// Return whether this [`AreaSubspace`] includes Entries with arbitrary subspace_ids. + pub fn is_any(&self) -> bool { + match self { + AreaSubspace::Any => true, + _ => false, + } + } } #[cfg(feature = "dev")] @@ -61,14 +69,48 @@ where #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct Area { /// To be included in this [`Area`], an [`Entry`]’s `subspace_id` must be equal to the subspace_id, unless it is any. - pub subspace: AreaSubspace, + subspace: AreaSubspace, /// To be included in this [`Area`], an [`Entry`]’s `path` must be prefixed by the path. - pub path: Path, + path: Path, /// To be included in this [`Area`], an [`Entry`]’s `timestamp` must be included in the times. - pub times: Range, + times: Range, } impl Area { + /// Create a new [`Area`]. + pub fn new( + subspace: AreaSubspace, + path: Path, + times: Range, + ) -> Self { + Area { + subspace, + path, + times, + } + } + + /// Return a reference to the [`AreaSubspace`]. + /// + /// To be included in this [`Area`], an [`Entry`]’s `subspace_id` must be equal to the subspace_id, unless it is any. + pub fn get_subspace(&self) -> &AreaSubspace { + &self.subspace + } + + /// Return a reference to the [`Path`]. + /// + /// To be included in this [`Area`], an [`Entry`]’s `path` must be prefixed by the path. + pub fn get_path(&self) -> &Path { + &self.path + } + + /// Return a reference to the range of [`Timestamp`]s. + /// + /// To be included in this [`Area`], an [`Entry`]’s `timestamp` must be included in the times. + pub fn get_times(&self) -> &Range { + &self.times + } + /// Return an [`Area`] which includes all entries. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#full_area). pub fn full() -> Self { @@ -94,9 +136,9 @@ impl Area, ) -> bool { - self.subspace.includes(&entry.subspace_id) - && self.path.is_prefix_of(&entry.path) - && self.times.includes(&entry.timestamp) + self.subspace.includes(entry.get_subspace_id()) + && self.path.is_prefix_of(entry.get_path()) + && self.times.includes(&entry.get_timestamp()) } /// Return whether an [`Area`] fully [includes](https://willowprotocol.org/specs/grouping-entries/index.html#area_include_area) another [`Area`]. diff --git a/data-model/src/grouping/range_3d.rs b/data-model/src/grouping/range_3d.rs index 94fb447..5c5db76 100644 --- a/data-model/src/grouping/range_3d.rs +++ b/data-model/src/grouping/range_3d.rs @@ -13,26 +13,57 @@ use crate::{ pub struct Range3d { /// A range of [`SubspaceId`]s. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#SubspaceRange). - pub subspaces: Range, + subspaces: Range, /// A range of [`Path`]s. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#PathRange). - pub paths: Range>, + paths: Range>, /// A range of [`Timestamp`]s. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#TimeRange). - pub times: Range, + times: Range, } impl Range3d { + /// Create a new [`Range3d`]. + pub fn new( + subspaces: Range, + paths: Range>, + times: Range, + ) -> Self { + Range3d { + subspaces, + paths, + times, + } + } + + /// Return a reference to the range of [`SubspaceId`]s. + /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#SubspaceRange). + pub fn get_subspaces(&self) -> &Range { + &self.subspaces + } + + /// Return a reference to the range of [`Path`]s. + /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#PathRange). + pub fn get_paths(&self) -> &Range> { + &self.paths + } + + /// Return a reference to the range of [`Timestamp`]s. + /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#TimeRange). + pub fn get_times(&self) -> &Range { + &self.times + } + /// Return whether an [`Entry`] is [included](https://willowprotocol.org/specs/grouping-entries/index.html#d3_range_include) by this 3d range. pub fn includes_entry( &self, entry: &Entry, ) -> bool { - self.subspaces.includes(&entry.subspace_id) - && self.paths.includes(&entry.path) - && self.times.includes(&entry.timestamp) + self.subspaces.includes(entry.get_subspace_id()) + && self.paths.includes(entry.get_path()) + && self.times.includes(&entry.get_timestamp()) } /// Return the intersection between this [`Range3d`] and another. @@ -113,14 +144,14 @@ mod tests { #[test] fn includes_entry() { - let entry = Entry { - namespace_id: FakeNamespaceId::default(), - subspace_id: FakeSubspaceId(10), - path: Path::::new_from_slice(&[Component::new(b"a").unwrap()]).unwrap(), - timestamp: 500, - payload_length: 10, - payload_digest: FakePayloadDigest::default(), - }; + let entry = Entry::new( + FakeNamespaceId::default(), + FakeSubspaceId(10), + Path::::new_from_slice(&[Component::new(b"a").unwrap()]).unwrap(), + 500, + 10, + FakePayloadDigest::default(), + ); let range_including = Range3d { subspaces: Range::::new_closed(FakeSubspaceId(9), FakeSubspaceId(11)) From 3dedad0d7785e4887bebad3842a705bfb1286ea8 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 12:09:48 +0200 Subject: [PATCH 36/40] Just clippy things --- data-model/src/encoding/relativity.rs | 16 ++++++++-------- data-model/src/grouping/area.rs | 5 +---- .../entry_rel_namespace_range_encoding.rs | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index aee0d57..19779e1 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -237,7 +237,7 @@ where } self.get_path() - .relative_encode(&reference.get_path(), consumer) + .relative_encode(reference.get_path(), consumer) .await?; encode_compact_width_be(time_diff, consumer).await?; @@ -309,7 +309,7 @@ where } */ - let path = Path::::relative_decode(&reference.get_path(), producer).await?; + let path = Path::::relative_decode(reference.get_path(), producer).await?; let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; @@ -405,7 +405,7 @@ where } self.get_path() - .relative_encode(&out.get_path(), consumer) + .relative_encode(out.get_path(), consumer) .await?; encode_compact_width_be(time_diff, consumer).await?; encode_compact_width_be(self.get_payload_length(), consumer).await?; @@ -458,9 +458,9 @@ where } }; - let path = Path::relative_decode(&out.get_path(), producer).await?; + let path = Path::relative_decode(out.get_path(), producer).await?; - if !path.is_prefixed_by(&out.get_path()) { + if !path.is_prefixed_by(out.get_path()) { return Err(DecodeError::InvalidInput); } @@ -814,7 +814,7 @@ where } self.get_path() - .relative_encode(&out.get_path(), consumer) + .relative_encode(out.get_path(), consumer) .await?; encode_compact_width_be(start_diff, consumer).await?; @@ -904,10 +904,10 @@ where } } - let path = Path::relative_decode(&out.get_path(), producer).await?; + let path = Path::relative_decode(out.get_path(), producer).await?; // Verify the decoded path is prefixed by the reference path - if !path.is_prefixed_by(&out.get_path()) { + if !path.is_prefixed_by(out.get_path()) { return Err(DecodeError::InvalidInput); } diff --git a/data-model/src/grouping/area.rs b/data-model/src/grouping/area.rs index 11849dd..1743ff9 100644 --- a/data-model/src/grouping/area.rs +++ b/data-model/src/grouping/area.rs @@ -39,10 +39,7 @@ impl AreaSubspace { /// Return whether this [`AreaSubspace`] includes Entries with arbitrary subspace_ids. pub fn is_any(&self) -> bool { - match self { - AreaSubspace::Any => true, - _ => false, - } + matches!(self, AreaSubspace::Any) } } diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs index c5a9053..013be9b 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs @@ -20,7 +20,7 @@ fuzz_target!(|data: ( return; } - let namespace = entry.namespace_id.clone(); + let namespace = entry.get_namespace_id().clone(); smol::block_on(async { relative_encoding_roundtrip::< From 9cf386a77d020afab25d798c1521165e3130abc3 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 12:17:20 +0200 Subject: [PATCH 37/40] *More* clippy things --- fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs | 2 +- fuzz/src/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs index 3b2439a..d2b0fb8 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs @@ -20,7 +20,7 @@ fuzz_target!(|data: ( return; } - let namespace = entry.namespace_id.clone(); + let namespace = entry.get_namespace_id().clone(); smol::block_on(async { relative_encoding_roundtrip::< diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index a3877e7..e24e54c 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,3 +1,4 @@ +#[allow(clippy::type_complexity)] pub mod encode; pub mod path; pub mod placeholder_params; From 2b3f44157d330e51e2c47bf40350fc43f64d3ddf Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 12:33:44 +0200 Subject: [PATCH 38/40] Silence clippy on type complexity in the workspace. Blessed silence. You are welcome. --- Cargo.toml | 2 ++ data-model/Cargo.toml | 3 ++- earthstar/Cargo.toml | 5 ++++- fuzz/Cargo.toml | 3 +++ fuzz/src/lib.rs | 1 - 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5821721..3d787a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,3 +3,5 @@ members = ["data-model", "earthstar", "fuzz"] resolver = "2" +[workspace.lints.clippy] +type_complexity = "allow" \ No newline at end of file diff --git a/data-model/Cargo.toml b/data-model/Cargo.toml index ac05cfe..e33f6c2 100644 --- a/data-model/Cargo.toml +++ b/data-model/Cargo.toml @@ -21,6 +21,7 @@ smol = "2.0.0" # document all features all-features = true - +[lints] +workspace = true diff --git a/earthstar/Cargo.toml b/earthstar/Cargo.toml index 130fa82..171cff7 100644 --- a/earthstar/Cargo.toml +++ b/earthstar/Cargo.toml @@ -7,4 +7,7 @@ edition = "2021" either = "1.10.0" willow-data-model = { path = "../data-model" } arbitrary = { version = "1.0.2", features = ["derive"]} -ufotofu = "0.3.0" \ No newline at end of file +ufotofu = "0.3.0" + +[lints] +workspace = true \ No newline at end of file diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index f776dfc..d13024e 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" [package.metadata] cargo-fuzz = true +[lints] +workspace = true + [dependencies] ufotofu = { version = "0.3.0", features=["dev"]} smol = "2.0.0" diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index e24e54c..a3877e7 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,4 +1,3 @@ -#[allow(clippy::type_complexity)] pub mod encode; pub mod path; pub mod placeholder_params; From 3b861e843ae5da271a8fdb145a30ff7ff2acd4b5 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 14:38:57 +0200 Subject: [PATCH 39/40] Shorten getter names --- data-model/src/encoding/relativity.rs | 326 +++++++++--------- data-model/src/entry.rs | 12 +- data-model/src/grouping/area.rs | 20 +- data-model/src/grouping/range_3d.rs | 12 +- .../entry_rel_namespace_area_encoding.rs | 2 +- .../entry_rel_namespace_range_encoding.rs | 2 +- 6 files changed, 187 insertions(+), 187 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 19779e1..1630e81 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -206,45 +206,45 @@ where where Consumer: BulkConsumer, { - let time_diff = self.get_timestamp().abs_diff(reference.get_timestamp()); + let time_diff = self.timestamp().abs_diff(reference.timestamp()); let mut header: u8 = 0b0000_0000; - if self.get_namespace_id() != reference.get_namespace_id() { + if self.namespace_id() != reference.namespace_id() { header |= 0b1000_0000; } - if self.get_subspace_id() != reference.get_subspace_id() { + if self.subspace_id() != reference.subspace_id() { header |= 0b0100_0000; } - if self.get_timestamp() > reference.get_timestamp() { + if self.timestamp() > reference.timestamp() { header |= 0b0010_0000; } header |= CompactWidth::from_u64(time_diff).bitmask(4); - header |= CompactWidth::from_u64(self.get_payload_length()).bitmask(6); + header |= CompactWidth::from_u64(self.payload_length()).bitmask(6); consumer.consume(header).await?; - if self.get_namespace_id() != reference.get_namespace_id() { - self.get_namespace_id().encode(consumer).await?; + if self.namespace_id() != reference.namespace_id() { + self.namespace_id().encode(consumer).await?; } - if self.get_subspace_id() != reference.get_subspace_id() { - self.get_subspace_id().encode(consumer).await?; + if self.subspace_id() != reference.subspace_id() { + self.subspace_id().encode(consumer).await?; } - self.get_path() - .relative_encode(reference.get_path(), consumer) + self.path() + .relative_encode(reference.path(), consumer) .await?; encode_compact_width_be(time_diff, consumer).await?; - encode_compact_width_be(self.get_payload_length(), consumer).await?; + encode_compact_width_be(self.payload_length(), consumer).await?; - self.get_payload_digest().encode(consumer).await?; + self.payload_digest().encode(consumer).await?; Ok(()) } @@ -284,7 +284,7 @@ where let namespace_id = if is_namespace_encoded { N::decode(producer).await? } else { - reference.get_namespace_id().clone() + reference.namespace_id().clone() }; /* @@ -298,7 +298,7 @@ where let subspace_id = if is_subspace_encoded { S::decode(producer).await? } else { - reference.get_subspace_id().clone() + reference.subspace_id().clone() }; /* @@ -309,19 +309,19 @@ where } */ - let path = Path::::relative_decode(reference.get_path(), producer).await?; + let path = Path::::relative_decode(reference.path(), producer).await?; let time_diff = decode_compact_width_be(compact_width_time_diff, producer).await?; // Add or subtract safely here to avoid overflows caused by malicious or faulty encodings. let timestamp = if add_or_subtract_time_diff { reference - .get_timestamp() + .timestamp() .checked_add(time_diff) .ok_or(DecodeError::InvalidInput)? } else { reference - .get_timestamp() + .timestamp() .checked_sub(time_diff) .ok_or(DecodeError::InvalidInput)? }; @@ -370,7 +370,7 @@ where { let (namespace, out) = reference; - if self.get_namespace_id() != namespace { + if self.namespace_id() != namespace { panic!("Tried to encode an entry relative to a namespace it does not belong to") } @@ -379,37 +379,37 @@ where } let time_diff = core::cmp::min( - self.get_timestamp() - out.get_times().start, - u64::from(&out.get_times().end) - self.get_timestamp(), + self.timestamp() - out.times().start, + u64::from(&out.times().end) - self.timestamp(), ); let mut header = 0b0000_0000; - if out.get_subspace().is_any() { + if out.subspace().is_any() { header |= 0b1000_0000; } - if self.get_timestamp() - out.get_times().start - <= u64::from(&out.get_times().end) - self.get_timestamp() + if self.timestamp() - out.times().start + <= u64::from(&out.times().end) - self.timestamp() { header |= 0b0100_0000; } header |= CompactWidth::from_u64(time_diff).bitmask(2); - header |= CompactWidth::from_u64(self.get_payload_length()).bitmask(4); + header |= CompactWidth::from_u64(self.payload_length()).bitmask(4); consumer.consume(header).await?; - if out.get_subspace().is_any() { - self.get_subspace_id().encode(consumer).await?; + if out.subspace().is_any() { + self.subspace_id().encode(consumer).await?; } - self.get_path() - .relative_encode(out.get_path(), consumer) + self.path() + .relative_encode(out.path(), consumer) .await?; encode_compact_width_be(time_diff, consumer).await?; - encode_compact_width_be(self.get_payload_length(), consumer).await?; - self.get_payload_digest().encode(consumer).await?; + encode_compact_width_be(self.payload_length(), consumer).await?; + self.payload_digest().encode(consumer).await?; Ok(()) } @@ -447,20 +447,20 @@ where } let subspace_id = if is_subspace_encoded { - match &out.get_subspace() { + match &out.subspace() { AreaSubspace::Any => S::decode(producer).await?, AreaSubspace::Id(_) => return Err(DecodeError::InvalidInput), } } else { - match &out.get_subspace() { + match &out.subspace() { AreaSubspace::Any => return Err(DecodeError::InvalidInput), AreaSubspace::Id(id) => id.clone(), } }; - let path = Path::relative_decode(out.get_path(), producer).await?; + let path = Path::relative_decode(out.path(), producer).await?; - if !path.is_prefixed_by(out.get_path()) { + if !path.is_prefixed_by(out.path()) { return Err(DecodeError::InvalidInput); } @@ -471,23 +471,23 @@ where let payload_digest = PD::decode(producer).await?; let timestamp = if add_time_diff_to_start { - out.get_times().start.checked_add(time_diff) + out.times().start.checked_add(time_diff) } else { - u64::from(&out.get_times().end).checked_sub(time_diff) + u64::from(&out.times().end).checked_sub(time_diff) } .ok_or(DecodeError::InvalidInput)?; // === Necessary to produce canonic encodings. === // Verify that the correct add_or_subtract_time_diff flag was set. - let should_have_added = timestamp.checked_sub(out.get_times().start) - <= u64::from(&out.get_times().end).checked_sub(timestamp); + let should_have_added = timestamp.checked_sub(out.times().start) + <= u64::from(&out.times().end).checked_sub(timestamp); if add_time_diff_to_start != should_have_added { return Err(DecodeError::InvalidInput); } // =============================================== - if !out.get_times().includes(×tamp) { + if !out.times().includes(×tamp) { return Err(DecodeError::InvalidInput); } @@ -522,7 +522,7 @@ where { let (namespace, out) = reference; - if self.get_namespace_id() != namespace { + if self.namespace_id() != namespace { panic!("Tried to encode an entry relative to a namespace it does not belong to") } @@ -531,25 +531,25 @@ where } let time_diff = core::cmp::min( - self.get_timestamp().abs_diff(out.get_times().start), - self.get_timestamp() - .abs_diff(u64::from(&out.get_times().end)), + self.timestamp().abs_diff(out.times().start), + self.timestamp() + .abs_diff(u64::from(&out.times().end)), ); let mut header = 0b0000_0000; // Encode e.get_subspace_id()? - if self.get_subspace_id() != &out.get_subspaces().start { + if self.subspace_id() != &out.subspaces().start { header |= 0b1000_0000; } // Encode e.get_path() relative to out.get_paths().start or to out.get_paths().end? - let encode_path_relative_to_start = match &out.get_paths().end { + let encode_path_relative_to_start = match &out.paths().end { RangeEnd::Closed(end_path) => { let start_lcp = self - .get_path() - .longest_common_prefix(&out.get_paths().start); - let end_lcp = self.get_path().longest_common_prefix(end_path); + .path() + .longest_common_prefix(&out.paths().start); + let end_lcp = self.path().longest_common_prefix(end_path); start_lcp.get_component_count() >= end_lcp.get_component_count() } @@ -562,7 +562,7 @@ where // Add time_diff to out.get_times().start, or subtract from out.get_times().end? let add_time_diff_with_start = - time_diff == self.get_timestamp().abs_diff(out.get_times().start); + time_diff == self.timestamp().abs_diff(out.times().start); if add_time_diff_with_start { header |= 0b0010_0000; @@ -572,35 +572,35 @@ where header |= CompactWidth::from_u64(time_diff).bitmask(4); // 2-bit integer n such that 2^n gives compact_width(e.get_payload_length()) - header |= CompactWidth::from_u64(self.get_payload_length()).bitmask(6); + header |= CompactWidth::from_u64(self.payload_length()).bitmask(6); consumer.consume(header).await?; - if self.get_subspace_id() != &out.get_subspaces().start { - self.get_subspace_id().encode(consumer).await?; + if self.subspace_id() != &out.subspaces().start { + self.subspace_id().encode(consumer).await?; } // Encode e.get_path() relative to out.get_paths().start or to out.get_paths().end? - match &out.get_paths().end { + match &out.paths().end { RangeEnd::Closed(end_path) => { if encode_path_relative_to_start { - self.get_path() - .relative_encode(&out.get_paths().start, consumer) + self.path() + .relative_encode(&out.paths().start, consumer) .await?; } else { - self.get_path().relative_encode(end_path, consumer).await?; + self.path().relative_encode(end_path, consumer).await?; } } RangeEnd::Open => { - self.get_path() - .relative_encode(&out.get_paths().start, consumer) + self.path() + .relative_encode(&out.paths().start, consumer) .await?; } } encode_compact_width_be(time_diff, consumer).await?; - encode_compact_width_be(self.get_payload_length(), consumer).await?; - self.get_payload_digest().encode(consumer).await?; + encode_compact_width_be(self.payload_length(), consumer).await?; + self.payload_digest().encode(consumer).await?; Ok(()) } @@ -647,40 +647,40 @@ where let subspace_id = if is_subspace_encoded { S::decode(producer).await? } else { - out.get_subspaces().start.clone() + out.subspaces().start.clone() }; // === Necessary to produce canonic encodings. === // Verify that encoding the subspace was necessary. - if subspace_id == out.get_subspaces().start && is_subspace_encoded { + if subspace_id == out.subspaces().start && is_subspace_encoded { return Err(DecodeError::InvalidInput); } // =============================================== // Verify that subspace is included by range - if !out.get_subspaces().includes(&subspace_id) { + if !out.subspaces().includes(&subspace_id) { return Err(DecodeError::InvalidInput); } let path = if decode_path_relative_to_start { - Path::relative_decode(&out.get_paths().start, producer).await? + Path::relative_decode(&out.paths().start, producer).await? } else { - match &out.get_paths().end { + match &out.paths().end { RangeEnd::Closed(end_path) => Path::relative_decode(end_path, producer).await?, RangeEnd::Open => return Err(DecodeError::InvalidInput), } }; // Verify that path is included by range - if !out.get_paths().includes(&path) { + if !out.paths().includes(&path) { return Err(DecodeError::InvalidInput); } // === Necessary to produce canonic encodings. === // Verify that the path was encoded relative to the correct bound of the referenc path range. - let should_have_encoded_path_relative_to_start = match &out.get_paths().end { + let should_have_encoded_path_relative_to_start = match &out.paths().end { RangeEnd::Closed(end_path) => { - let start_lcp = path.longest_common_prefix(&out.get_paths().start); + let start_lcp = path.longest_common_prefix(&out.paths().start); let end_lcp = path.longest_common_prefix(end_path); start_lcp.get_component_count() >= end_lcp.get_component_count() @@ -701,25 +701,25 @@ where let payload_digest = PD::decode(producer).await?; let timestamp = if add_time_diff_with_start { - out.get_times().start.checked_add(time_diff) + out.times().start.checked_add(time_diff) } else { - match &out.get_times().end { + match &out.times().end { RangeEnd::Closed(end_time) => end_time.checked_sub(time_diff), - RangeEnd::Open => u64::from(&out.get_times().end).checked_sub(time_diff), + RangeEnd::Open => u64::from(&out.times().end).checked_sub(time_diff), } } .ok_or(DecodeError::InvalidInput)?; // Verify that timestamp is included by range - if !out.get_times().includes(×tamp) { + if !out.times().includes(×tamp) { return Err(DecodeError::InvalidInput); } // === Necessary to produce canonic encodings. === // Verify that time_diff is what it should have been let correct_time_diff = core::cmp::min( - timestamp.abs_diff(out.get_times().start), - timestamp.abs_diff(u64::from(&out.get_times().end)), + timestamp.abs_diff(out.times().start), + timestamp.abs_diff(u64::from(&out.times().end)), ); if time_diff != correct_time_diff { @@ -727,7 +727,7 @@ where } // Verify that the combine with start bitflag in the header was correct - let should_have_added_to_start = time_diff == timestamp.abs_diff(out.get_times().start); + let should_have_added_to_start = time_diff == timestamp.abs_diff(out.times().start); if should_have_added_to_start != add_time_diff_with_start { return Err(DecodeError::InvalidInput); @@ -766,31 +766,31 @@ where } let start_diff = core::cmp::min( - self.get_times().start - out.get_times().start, - u64::from(&out.get_times().end) - self.get_times().start, + self.times().start - out.times().start, + u64::from(&out.times().end) - self.times().start, ); let end_diff = core::cmp::min( - u64::from(&self.get_times().end) - out.get_times().start, - u64::from(&out.get_times().end) - u64::from(&self.get_times().end), + u64::from(&self.times().end) - out.times().start, + u64::from(&out.times().end) - u64::from(&self.times().end), ); let mut header = 0; - if self.get_subspace() != out.get_subspace() { + if self.subspace() != out.subspace() { header |= 0b1000_0000; } - if self.get_times().end == RangeEnd::Open { + if self.times().end == RangeEnd::Open { header |= 0b0100_0000; } - if start_diff == self.get_times().start - out.get_times().start { + if start_diff == self.times().start - out.times().start { header |= 0b0010_0000; } - if self.get_times().end != RangeEnd::Open - && end_diff == u64::from(&self.get_times().end) - out.get_times().start + if self.times().end != RangeEnd::Open + && end_diff == u64::from(&self.times().end) - out.times().start { header |= 0b0001_0000; } @@ -800,7 +800,7 @@ where consumer.consume(header).await?; - match (&self.get_subspace(), &out.get_subspace()) { + match (&self.subspace(), &out.subspace()) { (AreaSubspace::Any, AreaSubspace::Any) => {} // Same subspace (AreaSubspace::Id(_), AreaSubspace::Id(_)) => {} // Same subspace (AreaSubspace::Id(subspace), AreaSubspace::Any) => { @@ -813,13 +813,13 @@ where } } - self.get_path() - .relative_encode(out.get_path(), consumer) + self.path() + .relative_encode(out.path(), consumer) .await?; encode_compact_width_be(start_diff, consumer).await?; - if self.get_times().end != RangeEnd::Open { + if self.times().end != RangeEnd::Open { encode_compact_width_be(end_diff, consumer).await?; } @@ -880,18 +880,18 @@ where // === Necessary to produce canonic encodings. === // Verify that subspace wasn't needlessly encoded - if &sub == out.get_subspace() { + if &sub == out.subspace() { return Err(DecodeError::InvalidInput); } // =============================================== sub } else { - out.get_subspace().clone() + out.subspace().clone() }; // Verify that the decoded subspace is included by the reference subspace - match (&out.get_subspace(), &subspace) { + match (&out.subspace(), &subspace) { (AreaSubspace::Any, AreaSubspace::Any) => {} (AreaSubspace::Any, AreaSubspace::Id(_)) => {} (AreaSubspace::Id(_), AreaSubspace::Any) => { @@ -904,26 +904,26 @@ where } } - let path = Path::relative_decode(out.get_path(), producer).await?; + let path = Path::relative_decode(out.path(), producer).await?; // Verify the decoded path is prefixed by the reference path - if !path.is_prefixed_by(out.get_path()) { + if !path.is_prefixed_by(out.path()) { return Err(DecodeError::InvalidInput); } let start_diff = decode_compact_width_be(start_diff_compact_width, producer).await?; let start = if add_start_diff { - out.get_times().start.checked_add(start_diff) + out.times().start.checked_add(start_diff) } else { - u64::from(&out.get_times().end).checked_sub(start_diff) + u64::from(&out.times().end).checked_sub(start_diff) } .ok_or(DecodeError::InvalidInput)?; // Verify they sent correct start diff let expected_start_diff = core::cmp::min( - start.checked_sub(out.get_times().start), - u64::from(&out.get_times().end).checked_sub(start), + start.checked_sub(out.times().start), + u64::from(&out.times().end).checked_sub(start), ) .ok_or(DecodeError::InvalidInput)?; @@ -935,7 +935,7 @@ where // Verify that bit 2 of the header was set correctly let should_add_start_diff = start_diff == start - .checked_sub(out.get_times().start) + .checked_sub(out.times().start) .ok_or(DecodeError::InvalidInput)?; if add_start_diff != should_add_start_diff { @@ -953,16 +953,16 @@ where let end_diff = decode_compact_width_be(end_diff_compact_width, producer).await?; let end = if add_end_diff { - out.get_times().start.checked_add(end_diff) + out.times().start.checked_add(end_diff) } else { - u64::from(&out.get_times().end).checked_sub(end_diff) + u64::from(&out.times().end).checked_sub(end_diff) } .ok_or(DecodeError::InvalidInput)?; // Verify they sent correct end diff let expected_end_diff = core::cmp::min( - end.checked_sub(out.get_times().start), - u64::from(&out.get_times().end).checked_sub(end), + end.checked_sub(out.times().start), + u64::from(&out.times().end).checked_sub(end), ) .ok_or(DecodeError::InvalidInput)?; @@ -973,7 +973,7 @@ where // === Necessary to produce canonic encodings. === let should_add_end_diff = end_diff == end - .checked_sub(out.get_times().start) + .checked_sub(out.times().start) .ok_or(DecodeError::InvalidInput)?; if add_end_diff != should_add_end_diff { @@ -987,7 +987,7 @@ where let times = Range { start, end }; // Verify the decoded time range is included by the reference time range - if !out.get_times().includes_range(×) { + if !out.times().includes_range(×) { return Err(DecodeError::InvalidInput); } @@ -1011,16 +1011,16 @@ where where Consumer: BulkConsumer, { - let start_to_start = self.get_times().start.abs_diff(reference.get_times().start); - let start_to_end = match reference.get_times().end { - RangeEnd::Closed(end) => self.get_times().start.abs_diff(end), + let start_to_start = self.times().start.abs_diff(reference.times().start); + let start_to_end = match reference.times().end { + RangeEnd::Closed(end) => self.times().start.abs_diff(end), RangeEnd::Open => u64::MAX, }; - let end_to_start = match self.get_times().end { - RangeEnd::Closed(end) => end.abs_diff(reference.get_times().start), + let end_to_start = match self.times().end { + RangeEnd::Closed(end) => end.abs_diff(reference.times().start), RangeEnd::Open => u64::MAX, }; - let end_to_end = match (&self.get_times().end, &reference.get_times().end) { + let end_to_end = match (&self.times().end, &reference.times().end) { (RangeEnd::Closed(self_end), RangeEnd::Closed(ref_end)) => self_end.abs_diff(*ref_end), (RangeEnd::Closed(_), RangeEnd::Open) => u64::MAX, (RangeEnd::Open, RangeEnd::Closed(_)) => u64::MAX, @@ -1034,32 +1034,32 @@ where let mut header_1 = 0b0000_0000; // Bits 0, 1 - Encode r.get_subspaces().start? - if self.get_subspaces().start == reference.get_subspaces().start { + if self.subspaces().start == reference.subspaces().start { header_1 |= 0b0100_0000; - } else if reference.get_subspaces().end == self.get_subspaces().start { + } else if reference.subspaces().end == self.subspaces().start { header_1 |= 0b1000_0000; } else { header_1 |= 0b1100_0000; } // Bits 2, 3 - Encode r.get_subspaces().end? - if self.get_subspaces().end == RangeEnd::Open { + if self.subspaces().end == RangeEnd::Open { // Do nothing - } else if self.get_subspaces().end == reference.get_subspaces().start { + } else if self.subspaces().end == reference.subspaces().start { header_1 |= 0b0001_0000; - } else if self.get_subspaces().end == reference.get_subspaces().end { + } else if self.subspaces().end == reference.subspaces().end { header_1 |= 0b0010_0000; - } else if self.get_subspaces().end != RangeEnd::Open { + } else if self.subspaces().end != RangeEnd::Open { header_1 |= 0b0011_0000; } // Bit 4 - Encode r.get_paths().start relative to ref.get_paths().start or to ref.get_paths().end? - if let RangeEnd::Closed(ref_path_end) = &reference.get_paths().end { + if let RangeEnd::Closed(ref_path_end) = &reference.paths().end { let lcp_start_start = self - .get_paths() + .paths() .start - .longest_common_prefix(&reference.get_paths().start); - let lcp_start_end = self.get_paths().start.longest_common_prefix(ref_path_end); + .longest_common_prefix(&reference.paths().start); + let lcp_start_end = self.paths().start.longest_common_prefix(ref_path_end); if lcp_start_start.get_component_count() >= lcp_start_end.get_component_count() { header_1 |= 0b0000_1000; @@ -1069,15 +1069,15 @@ where } // Bit 5 - Self path end open? - if self.get_paths().end == RangeEnd::Open { + if self.paths().end == RangeEnd::Open { header_1 |= 0b0000_0100; } // Bit 6 - Encode r.get_paths().end relative to ref.get_paths().start or to ref.get_paths().end (if at all)? - match (&self.get_paths().end, &reference.get_paths().end) { + match (&self.paths().end, &reference.paths().end) { (RangeEnd::Closed(self_path_end), RangeEnd::Closed(ref_path_end)) => { let lcp_end_start = - self_path_end.longest_common_prefix(&reference.get_paths().start); + self_path_end.longest_common_prefix(&reference.paths().start); let lcp_end_end = self_path_end.longest_common_prefix(ref_path_end); if lcp_end_start.get_component_count() > lcp_end_end.get_component_count() { @@ -1092,7 +1092,7 @@ where } // Bit 7 - Self time end open? - if self.get_times().end == RangeEnd::Open { + if self.times().end == RangeEnd::Open { header_1 |= 0b0000_0001; } @@ -1106,8 +1106,8 @@ where } // Bit 9 -Add or subtract start_time_diff? - if is_bitflagged(header_2, 0) && self.get_times().start >= reference.get_times().start - || !is_bitflagged(header_2, 0) && self.get_times().start >= reference.get_times().end + if is_bitflagged(header_2, 0) && self.times().start >= reference.times().start + || !is_bitflagged(header_2, 0) && self.times().start >= reference.times().end { header_2 |= 0b0100_0000; } @@ -1116,22 +1116,22 @@ where header_2 |= CompactWidth::from_u64(start_time_diff).bitmask(2); // Bit 12 - Encode r.get_times().end relative to ref.get_times().start or ref.get_times().end (if at all)? - if self.get_times().end != RangeEnd::Open && end_to_start <= end_to_end { + if self.times().end != RangeEnd::Open && end_to_start <= end_to_end { header_2 |= 0b0000_1000; } // Bit 13 - Add or subtract end_time_diff (if encoding it at all)? - if self.get_times().end == RangeEnd::Open { + if self.times().end == RangeEnd::Open { // do nothing } else if (is_bitflagged(header_2, 4) - && self.get_times().end >= reference.get_times().start) - || (!is_bitflagged(header_2, 4) && self.get_times().end >= reference.get_times().end) + && self.times().end >= reference.times().start) + || (!is_bitflagged(header_2, 4) && self.times().end >= reference.times().end) { header_2 |= 0b0000_0100; } // Bits 14, 15 - ignored, or 2-bit integer n such that 2^n gives compact_width(end_time_diff) - if self.get_times().end == RangeEnd::Open { + if self.times().end == RangeEnd::Open { // do nothing } else { header_2 |= CompactWidth::from_u64(end_time_diff).bitmask(6); @@ -1139,41 +1139,41 @@ where consumer.consume(header_2).await?; - if (self.get_subspaces().start == reference.get_subspaces().start) - || (reference.get_subspaces().end == self.get_subspaces().start) + if (self.subspaces().start == reference.subspaces().start) + || (reference.subspaces().end == self.subspaces().start) { // Don't encode } else { - self.get_subspaces().start.encode(consumer).await?; + self.subspaces().start.encode(consumer).await?; } - if self.get_subspaces().end == RangeEnd::Open - || (self.get_subspaces().end == reference.get_subspaces().start) - || (self.get_subspaces().end == reference.get_subspaces().end) + if self.subspaces().end == RangeEnd::Open + || (self.subspaces().end == reference.subspaces().start) + || (self.subspaces().end == reference.subspaces().end) { // Don't encode end subspace - } else if let RangeEnd::Closed(end_subspace) = &self.get_subspaces().end { + } else if let RangeEnd::Closed(end_subspace) = &self.subspaces().end { end_subspace.encode(consumer).await?; } if is_bitflagged(header_1, 4) { - self.get_paths() + self.paths() .start - .relative_encode(&reference.get_paths().start, consumer) + .relative_encode(&reference.paths().start, consumer) .await?; - } else if let RangeEnd::Closed(end_path) = &reference.get_paths().end { - self.get_paths() + } else if let RangeEnd::Closed(end_path) = &reference.paths().end { + self.paths() .start .relative_encode(end_path, consumer) .await?; } - if let RangeEnd::Closed(end_path) = &self.get_paths().end { + if let RangeEnd::Closed(end_path) = &self.paths().end { if is_bitflagged(header_1, 6) { end_path - .relative_encode(&reference.get_paths().start, consumer) + .relative_encode(&reference.paths().start, consumer) .await? - } else if let RangeEnd::Closed(ref_end_path) = &reference.get_paths().end { + } else if let RangeEnd::Closed(ref_end_path) = &reference.paths().end { end_path.relative_encode(ref_end_path, consumer).await?; } } @@ -1221,8 +1221,8 @@ where // Decode subspace start let subspace_start = match subspace_start_flags { - 0b0100_0000 => reference.get_subspaces().start.clone(), - 0b1000_0000 => match &reference.get_subspaces().end { + 0b0100_0000 => reference.subspaces().start.clone(), + 0b1000_0000 => match &reference.subspaces().end { RangeEnd::Closed(end) => end.clone(), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, @@ -1232,8 +1232,8 @@ where let subspace_end = match subspace_end_flags { 0b0000_0000 => RangeEnd::Open, - 0b0001_0000 => RangeEnd::Closed(reference.get_subspaces().start.clone()), - 0b0010_0000 => match &reference.get_subspaces().end { + 0b0001_0000 => RangeEnd::Closed(reference.subspaces().start.clone()), + 0b0010_0000 => match &reference.subspaces().end { RangeEnd::Closed(end) => RangeEnd::Closed(end.clone()), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, @@ -1241,12 +1241,12 @@ where _ => RangeEnd::Closed(S::decode(producer).await?), }; - let path_start = match (is_path_start_rel_to_start, &reference.get_paths().end) { + let path_start = match (is_path_start_rel_to_start, &reference.paths().end) { (true, RangeEnd::Closed(_)) => { - Path::relative_decode(&reference.get_paths().start, producer).await? + Path::relative_decode(&reference.paths().start, producer).await? } (true, RangeEnd::Open) => { - Path::relative_decode(&reference.get_paths().start, producer).await? + Path::relative_decode(&reference.paths().start, producer).await? } (false, RangeEnd::Closed(path_end)) => { Path::relative_decode(path_end, producer).await? @@ -1257,9 +1257,9 @@ where let path_end = if is_path_end_open { RangeEnd::Open } else if is_path_end_rel_to_start { - RangeEnd::Closed(Path::relative_decode(&reference.get_paths().start, producer).await?) + RangeEnd::Closed(Path::relative_decode(&reference.paths().start, producer).await?) } else { - match &reference.get_paths().end { + match &reference.paths().end { RangeEnd::Closed(end) => { RangeEnd::Closed(Path::relative_decode(end, producer).await?) } @@ -1271,13 +1271,13 @@ where decode_compact_width_be(start_time_diff_compact_width, producer).await?; let time_start = match (is_time_start_rel_to_start, add_or_subtract_start_time_diff) { - (true, true) => reference.get_times().start.checked_add(start_time_diff), - (true, false) => reference.get_times().start.checked_sub(start_time_diff), - (false, true) => match reference.get_times().end { + (true, true) => reference.times().start.checked_add(start_time_diff), + (true, false) => reference.times().start.checked_sub(start_time_diff), + (false, true) => match reference.times().end { RangeEnd::Closed(ref_end) => ref_end.checked_add(start_time_diff), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, - (false, false) => match reference.get_times().end { + (false, false) => match reference.times().end { RangeEnd::Closed(ref_end) => ref_end.checked_sub(start_time_diff), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, @@ -1292,19 +1292,19 @@ where match (is_times_end_rel_to_start, add_or_subtract_end_time_diff) { (true, true) => RangeEnd::Closed( reference - .get_times() + .times() .start .checked_add(end_time_diff) .ok_or(DecodeError::InvalidInput)?, ), (true, false) => RangeEnd::Closed( reference - .get_times() + .times() .start .checked_sub(end_time_diff) .ok_or(DecodeError::InvalidInput)?, ), - (false, true) => match reference.get_times().end { + (false, true) => match reference.times().end { RangeEnd::Closed(ref_end) => RangeEnd::Closed( ref_end .checked_add(end_time_diff) @@ -1312,7 +1312,7 @@ where ), RangeEnd::Open => Err(DecodeError::InvalidInput)?, }, - (false, false) => match reference.get_times().end { + (false, false) => match reference.times().end { RangeEnd::Closed(ref_end) => RangeEnd::Closed( ref_end .checked_sub(end_time_diff) diff --git a/data-model/src/entry.rs b/data-model/src/entry.rs index 7933e53..5f50f8d 100644 --- a/data-model/src/entry.rs +++ b/data-model/src/entry.rs @@ -75,32 +75,32 @@ where } /// Return a reference to the identifier of the namespace to which the [`Entry`] belongs. - pub fn get_namespace_id(&self) -> &N { + pub fn namespace_id(&self) -> &N { &self.namespace_id } /// Return a reference to the identifier of the subspace_id to which the [`Entry`] belongs. - pub fn get_subspace_id(&self) -> &S { + pub fn subspace_id(&self) -> &S { &self.subspace_id } /// Return a reference to the [`Path`] to which the [`Entry`] was written. - pub fn get_path(&self) -> &Path { + pub fn path(&self) -> &Path { &self.path } /// Return the claimed creation time of the [`Entry`]. - pub fn get_timestamp(&self) -> Timestamp { + pub fn timestamp(&self) -> Timestamp { self.timestamp } /// Return the length of the Payload in bytes. - pub fn get_payload_length(&self) -> u64 { + pub fn payload_length(&self) -> u64 { self.payload_length } /// Return a reference to the result of applying hash_payload to the Payload. - pub fn get_payload_digest(&self) -> &PD { + pub fn payload_digest(&self) -> &PD { &self.payload_digest } diff --git a/data-model/src/grouping/area.rs b/data-model/src/grouping/area.rs index 1743ff9..04dc1a5 100644 --- a/data-model/src/grouping/area.rs +++ b/data-model/src/grouping/area.rs @@ -90,27 +90,27 @@ impl Area &AreaSubspace { + pub fn subspace(&self) -> &AreaSubspace { &self.subspace } /// Return a reference to the [`Path`]. /// /// To be included in this [`Area`], an [`Entry`]’s `path` must be prefixed by the path. - pub fn get_path(&self) -> &Path { + pub fn path(&self) -> &Path { &self.path } /// Return a reference to the range of [`Timestamp`]s. /// /// To be included in this [`Area`], an [`Entry`]’s `timestamp` must be included in the times. - pub fn get_times(&self) -> &Range { + pub fn times(&self) -> &Range { &self.times } /// Return an [`Area`] which includes all entries. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#full_area). - pub fn full() -> Self { + pub fn new_full() -> Self { Self { subspace: AreaSubspace::Any, path: Path::new_empty(), @@ -120,7 +120,7 @@ impl Area Self { + pub fn new_subspace(sub: S) -> Self { Self { subspace: AreaSubspace::Id(sub), path: Path::new_empty(), @@ -133,9 +133,9 @@ impl Area, ) -> bool { - self.subspace.includes(entry.get_subspace_id()) - && self.path.is_prefix_of(entry.get_path()) - && self.times.includes(&entry.get_timestamp()) + self.subspace.includes(entry.subspace_id()) + && self.path.is_prefix_of(entry.path()) + && self.times.includes(&entry.timestamp()) } /// Return whether an [`Area`] fully [includes](https://willowprotocol.org/specs/grouping-entries/index.html#area_include_area) another [`Area`]. @@ -267,7 +267,7 @@ mod tests { #[test] fn area_full() { - let full_area = Area::::full(); + let full_area = Area::::new_full(); assert_eq!( full_area, @@ -281,7 +281,7 @@ mod tests { #[test] fn area_subspace() { - let subspace_area = Area::::subspace(FakeSubspaceId(7)); + let subspace_area = Area::::new_subspace(FakeSubspaceId(7)); assert_eq!( subspace_area, diff --git a/data-model/src/grouping/range_3d.rs b/data-model/src/grouping/range_3d.rs index 5c5db76..d77c59c 100644 --- a/data-model/src/grouping/range_3d.rs +++ b/data-model/src/grouping/range_3d.rs @@ -40,19 +40,19 @@ impl /// Return a reference to the range of [`SubspaceId`]s. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#SubspaceRange). - pub fn get_subspaces(&self) -> &Range { + pub fn subspaces(&self) -> &Range { &self.subspaces } /// Return a reference to the range of [`Path`]s. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#PathRange). - pub fn get_paths(&self) -> &Range> { + pub fn paths(&self) -> &Range> { &self.paths } /// Return a reference to the range of [`Timestamp`]s. /// [Definition](https://willowprotocol.org/specs/grouping-entries/index.html#TimeRange). - pub fn get_times(&self) -> &Range { + pub fn times(&self) -> &Range { &self.times } @@ -61,9 +61,9 @@ impl &self, entry: &Entry, ) -> bool { - self.subspaces.includes(entry.get_subspace_id()) - && self.paths.includes(entry.get_path()) - && self.times.includes(&entry.get_timestamp()) + self.subspaces.includes(entry.subspace_id()) + && self.paths.includes(entry.path()) + && self.times.includes(&entry.timestamp()) } /// Return the intersection between this [`Range3d`] and another. diff --git a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs index d2b0fb8..596b3d9 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_area_encoding.rs @@ -20,7 +20,7 @@ fuzz_target!(|data: ( return; } - let namespace = entry.get_namespace_id().clone(); + let namespace = entry.namespace_id().clone(); smol::block_on(async { relative_encoding_roundtrip::< diff --git a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs index 013be9b..9de40fe 100644 --- a/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs +++ b/fuzz/fuzz_targets/entry_rel_namespace_range_encoding.rs @@ -20,7 +20,7 @@ fuzz_target!(|data: ( return; } - let namespace = entry.get_namespace_id().clone(); + let namespace = entry.namespace_id().clone(); smol::block_on(async { relative_encoding_roundtrip::< From a6f493db114cbce5cd27cd3c2881d663bae54ab8 Mon Sep 17 00:00:00 2001 From: Aljoscha Meyer Date: Wed, 24 Jul 2024 14:42:20 +0200 Subject: [PATCH 40/40] Run cargo fmt --- data-model/src/encoding/relativity.rs | 28 ++++++++------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/data-model/src/encoding/relativity.rs b/data-model/src/encoding/relativity.rs index 1630e81..d814dd3 100644 --- a/data-model/src/encoding/relativity.rs +++ b/data-model/src/encoding/relativity.rs @@ -389,9 +389,7 @@ where header |= 0b1000_0000; } - if self.timestamp() - out.times().start - <= u64::from(&out.times().end) - self.timestamp() - { + if self.timestamp() - out.times().start <= u64::from(&out.times().end) - self.timestamp() { header |= 0b0100_0000; } @@ -404,9 +402,7 @@ where self.subspace_id().encode(consumer).await?; } - self.path() - .relative_encode(out.path(), consumer) - .await?; + self.path().relative_encode(out.path(), consumer).await?; encode_compact_width_be(time_diff, consumer).await?; encode_compact_width_be(self.payload_length(), consumer).await?; self.payload_digest().encode(consumer).await?; @@ -532,8 +528,7 @@ where let time_diff = core::cmp::min( self.timestamp().abs_diff(out.times().start), - self.timestamp() - .abs_diff(u64::from(&out.times().end)), + self.timestamp().abs_diff(u64::from(&out.times().end)), ); let mut header = 0b0000_0000; @@ -546,9 +541,7 @@ where // Encode e.get_path() relative to out.get_paths().start or to out.get_paths().end? let encode_path_relative_to_start = match &out.paths().end { RangeEnd::Closed(end_path) => { - let start_lcp = self - .path() - .longest_common_prefix(&out.paths().start); + let start_lcp = self.path().longest_common_prefix(&out.paths().start); let end_lcp = self.path().longest_common_prefix(end_path); start_lcp.get_component_count() >= end_lcp.get_component_count() @@ -561,8 +554,7 @@ where } // Add time_diff to out.get_times().start, or subtract from out.get_times().end? - let add_time_diff_with_start = - time_diff == self.timestamp().abs_diff(out.times().start); + let add_time_diff_with_start = time_diff == self.timestamp().abs_diff(out.times().start); if add_time_diff_with_start { header |= 0b0010_0000; @@ -813,9 +805,7 @@ where } } - self.path() - .relative_encode(out.path(), consumer) - .await?; + self.path().relative_encode(out.path(), consumer).await?; encode_compact_width_be(start_diff, consumer).await?; @@ -1076,8 +1066,7 @@ where // Bit 6 - Encode r.get_paths().end relative to ref.get_paths().start or to ref.get_paths().end (if at all)? match (&self.paths().end, &reference.paths().end) { (RangeEnd::Closed(self_path_end), RangeEnd::Closed(ref_path_end)) => { - let lcp_end_start = - self_path_end.longest_common_prefix(&reference.paths().start); + let lcp_end_start = self_path_end.longest_common_prefix(&reference.paths().start); let lcp_end_end = self_path_end.longest_common_prefix(ref_path_end); if lcp_end_start.get_component_count() > lcp_end_end.get_component_count() { @@ -1123,8 +1112,7 @@ where // Bit 13 - Add or subtract end_time_diff (if encoding it at all)? if self.times().end == RangeEnd::Open { // do nothing - } else if (is_bitflagged(header_2, 4) - && self.times().end >= reference.times().start) + } else if (is_bitflagged(header_2, 4) && self.times().end >= reference.times().start) || (!is_bitflagged(header_2, 4) && self.times().end >= reference.times().end) { header_2 |= 0b0000_0100;