Merge pull request #24 from earthstar-project/relative-encodings

Relative encodings

Cargo.toml

@@ -3,3 +3,5 @@
 members = ["data-model", "earthstar", "fuzz"]
 resolver = "2"
 
+[workspace.lints.clippy]
+type_complexity = "allow"

data-model/Cargo.toml

@@ -10,7 +10,7 @@ dev = ["dep:arbitrary"]
 [dependencies]
 either = "1.10.0"
 arbitrary = { version = "1.0.2", features = ["derive"], optional = true }
-ufotofu = "0.2.0"
+ufotofu = "0.3.0"
 bytes = "1.6.0"
 
 [dev-dependencies]
@@ -21,6 +21,7 @@ smol = "2.0.0"
 # document all features
 all-features = true
 
-
+[lints]
+workspace = true
 
 

data-model/src/encoding/bytes.rs

@@ -0,0 +1,34 @@
+use either::Either;
+use ufotofu::local_nb::BulkProducer;
+
+use crate::encoding::error::DecodeError;
+
+/// Have `Producer` produce a single byte, or return an error if the final value was produced or the producer experienced an error.
+pub async fn produce_byte<Producer>(
+    producer: &mut Producer,
+) -> Result<u8, DecodeError<Producer::Error>>
+where
+    Producer: BulkProducer<Item = u8>,
+{
+    match producer.produce().await {
+        Ok(Either::Left(item)) => Ok(item),
+        Ok(Either::Right(_)) => Err(DecodeError::InvalidInput),
+        Err(err) => Err(DecodeError::Producer(err)),
+    }
+}
+
+pub fn is_bitflagged(byte: u8, position: u8) -> bool {
+    let mask = match position {
+        0 => 0b1000_0000,
+        1 => 0b0100_0000,
+        2 => 0b0010_0000,
+        3 => 0b0001_0000,
+        4 => 0b0000_1000,
+        5 => 0b0000_0100,
+        6 => 0b0000_0010,
+        7 => 0b0000_0001,
+        _ => panic!("Can't check for a bitflag at a position greater than 7"),
+    };
+
+    byte & mask == mask
+}

data-model/src/encoding/compact_width.rs

@@ -1,7 +1,5 @@
-use core::error::Error;
-
-use crate::encoding::error::{DecodeError, EncodingConsumerError};
-use crate::encoding::parameters::Decoder;
+use crate::encoding::error::DecodeError;
+use crate::encoding::parameters::Decodable;
 use crate::encoding::unsigned_int::{U16BE, U32BE, U64BE, U8BE};
 use ufotofu::local_nb::{BulkConsumer, BulkProducer};
 
@@ -88,21 +86,39 @@ impl CompactWidth {
             CompactWidth::Eight => 8,
         }
     }
+
+    /// Encode a [`CompactWidth`] as a 2-bit integer `n` such that 2^n gives the bytewidth of the [`CompactWidth`], and then place that 2-bit number into a `u8` at the bit-index of `position`.
+    pub fn bitmask(&self, position: u8) -> u8 {
+        let og = match self {
+            CompactWidth::One => 0b0000_0000,
+            CompactWidth::Two => 0b0100_0000,
+            CompactWidth::Four => 0b1000_0000,
+            CompactWidth::Eight => 0b1100_0000,
+        };
+
+        og >> position
+    }
+
+    pub fn decode_fixed_width_bitmask(mask: u8, offset: u8) -> Self {
+        let twobit_mask = 0b0000_0011;
+        let two_bit_int = mask >> (6 - offset) & twobit_mask;
+
+        // Because we sanitise the input down to a 2-bit integer, we can safely unwrap this.
+        CompactWidth::new(2u8.pow(two_bit_int as u32)).unwrap()
+    }
 }
 
 /// Encode a `u64` integer as a `compact_width(value)`-byte big-endian integer, and consume that with a [`BulkConsumer`].
 pub async fn encode_compact_width_be<Consumer: BulkConsumer<Item = u8>>(
     value: u64,
     consumer: &mut Consumer,
-) -> Result<(), EncodingConsumerError<Consumer::Error>>
-where
-    Consumer::Error: Error,
-{
+) -> Result<(), Consumer::Error> {
     let width = CompactWidth::from_u64(value).width();
 
     consumer
         .bulk_consume_full_slice(&value.to_be_bytes()[8 - width..])
-        .await?;
+        .await
+        .map_err(|f| f.reason)?;
 
     Ok(())
 }
@@ -112,18 +128,26 @@ pub async fn decode_compact_width_be<Producer: BulkProducer<Item = u8>>(
     compact_width: CompactWidth,
     producer: &mut Producer,
 ) -> Result<u64, DecodeError<Producer::Error>> {
-    match compact_width {
+    let decoded = match compact_width {
         CompactWidth::One => U8BE::decode(producer).await.map(u64::from),
         CompactWidth::Two => U16BE::decode(producer).await.map(u64::from),
         CompactWidth::Four => U32BE::decode(producer).await.map(u64::from),
         CompactWidth::Eight => U64BE::decode(producer).await.map(u64::from),
+    }?;
+
+    let real_width = CompactWidth::from_u64(decoded);
+
+    if real_width != compact_width {
+        return Err(DecodeError::InvalidInput);
     }
+
+    Ok(decoded)
 }
 
 #[cfg(test)]
 mod tests {
     use ufotofu::local_nb::consumer::IntoVec;
-    use ufotofu::local_nb::producer::FromVec;
+    use ufotofu::local_nb::producer::FromBoxedSlice;
 
     use super::*;
 
@@ -207,7 +231,7 @@ mod tests {
 
                 assert_eq!(decoded_compact_width, compact_width);
 
-                let mut producer = FromVec::new(encode_result);
+                let mut producer = FromBoxedSlice::from_vec(encode_result);
 
                 let decode_result = decode_compact_width_be(decoded_compact_width, &mut producer)
                     .await

data-model/src/encoding/error.rs

@@ -1,44 +1,7 @@
 use core::error::Error;
 use core::{fmt::Display, fmt::Formatter, num::TryFromIntError};
 use either::Either;
-use ufotofu::common::errors::{ConsumeFullSliceError, OverwriteFullSliceError};
-
-/// Returned when a encoding fails to be consumed by a [`ufotofu::local_nb::Consumer`].
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct EncodingConsumerError<E> {
-    /// The number of bytes which were consumed before the error.
-    pub bytes_consumed: usize,
-    /// The error returned on the final and failed attempt to consume bytes.
-    pub reason: E,
-}
-
-impl<E> From<ConsumeFullSliceError<E>> for EncodingConsumerError<E> {
-    fn from(err: ConsumeFullSliceError<E>) -> Self {
-        EncodingConsumerError {
-            bytes_consumed: err.consumed,
-            reason: err.reason,
-        }
-    }
-}
-
-impl<E> Error for EncodingConsumerError<E>
-where
-    E: 'static + Error,
-{
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        Some(&self.reason)
-    }
-}
-
-impl<E> Display for EncodingConsumerError<E> {
-    fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {
-        write!(
-            f,
-            "The consumer failed to consume after consuming {} bytes",
-            self.bytes_consumed
-        )
-    }
-}
+use ufotofu::common::errors::OverwriteFullSliceError;
 
 /// Everything that can go wrong when decoding a value.
 #[derive(Debug, Clone, PartialEq, Eq)]

data-model/src/encoding/max_power.rs

@@ -1,7 +1,7 @@
 use core::mem::size_of;
 use ufotofu::local_nb::{BulkConsumer, BulkProducer};
 
-use crate::encoding::error::{DecodeError, EncodingConsumerError};
+use crate::encoding::error::DecodeError;
 
 /// Return the least natural number such that 256^`n` is greater than or equal to `n`.
 ///
@@ -31,7 +31,7 @@ pub async fn encode_max_power<C>(
     value: usize,
     max_size: usize,
     consumer: &mut C,
-) -> Result<(), EncodingConsumerError<C::Error>>
+) -> Result<(), C::Error>
 where
     C: BulkConsumer<Item = u8>,
 {
@@ -44,7 +44,8 @@ where
 
     consumer
         .bulk_consume_full_slice(&value_encoded_raw[size_of::<u64>() - (power as usize)..])
-        .await?;
+        .await
+        .map_err(|f| f.reason)?;
 
     Ok(())
 }

data-model/src/encoding/mod.rs

@@ -1,5 +1,8 @@
+pub mod bytes;
 pub mod compact_width;
 pub mod error;
 pub mod max_power;
 pub mod parameters;
+pub mod relativity;
+pub(crate) mod shared_buffers;
 pub mod unsigned_int;

data-model/src/encoding/parameters.rs

@@ -1,27 +1,27 @@
 use std::future::Future;
 
-use crate::encoding::error::{DecodeError, EncodingConsumerError};
+use crate::encoding::error::DecodeError;
 use ufotofu::local_nb::{BulkConsumer, BulkProducer};
 
 /// A type that can be encoded to a bytestring, ensuring that any value of `Self` maps to exactly one bytestring.
 ///
 /// [Definition](https://willowprotocol.org/specs/encodings/index.html#encodings_what)
-pub trait Encoder {
+pub trait Encodable {
     /// A function from the set `Self` to the set of bytestrings.
     ///
     /// [Definition](https://willowprotocol.org/specs/encodings/index.html#encode_s)
     fn encode<Consumer>(
         &self,
         consumer: &mut Consumer,
-    ) -> impl Future<Output = Result<(), EncodingConsumerError<Consumer::Error>>>
+    ) -> impl Future<Output = Result<(), Consumer::Error>>
     where
         Consumer: BulkConsumer<Item = u8>;
 }
 
 /// A type that can be decoded from a bytestring, ensuring that every valid encoding maps to exactly one member of `Self`.
 ///
 /// [Definition](https://willowprotocol.org/specs/encodings/index.html#encodings_what)
-pub trait Decoder {
+pub trait Decodable {
     /// A function from the set of bytestrings to the set of `T`.
     ///
     /// [Definition](https://willowprotocol.org/specs/encodings/index.html#decode_s)