update encoding/decoding API to split off witness data

fuzz/fuzz_targets/c_rust_merkle.rs

@@ -8,23 +8,29 @@ use simplicity::jet::Elements;
 use simplicity::{BitIter, RedeemNode};
 
 fn do_test(data: &[u8]) {
-    if data.len() < 4 {
-        return;
-    }
-    let prog_len = (usize::from(data[0]) << 24)
-        + (usize::from(data[1]) << 16)
-        + (usize::from(data[2]) << 8)
-        + usize::from(data[3]);
-    if prog_len >= data.len() {
-        return;
-    }
-    let data = &data[4..];
+    // To decode the program length, we first try decoding the program using
+    // `decode_expression` which will not error on a length check. Alternately
+    // we could decode using RedeemNode::decode and then extract the length
+    // from the error return.
+    //
+    // If the program doesn't decode, just use the first byte as a "length".
+    let prog_len = {
+        let mut iter = BitIter::from(data);
+        match simplicity::decode::decode_expression::<_, Elements>(&mut iter) {
+            Ok(_) => (iter.n_total_read() + 7) / 8,
+            Err(_) => match data.first() {
+                Some(&n) => core::cmp::min(data.len(), n.into()),
+                None => return,
+            },
+        }
+    };
 
     let (program, witness) = data.split_at(prog_len);
     let c_result = run_program(program, witness, TestUpTo::CheckOneOne);
 
-    let mut iter = BitIter::from(program);
-    let rust_result = RedeemNode::<Elements>::decode(&mut iter);
+    let prog_iter = BitIter::from(program);
+    let wit_iter = BitIter::from(witness);
+    let rust_result = RedeemNode::<Elements>::decode(prog_iter, wit_iter);
 
     match (c_result, rust_result) {
         (Ok(_), Err(e)) => panic!("C accepted code that Rust rejected: {}", e),
@@ -78,7 +84,7 @@ mod tests {
     #[test]
     fn duplicate_crash() {
         let mut a = Vec::new();
-        extend_vec_from_hex("d73804000000000000000000000002040404040000000000000040000001ff0000000800380000000000000002040404040000000000000040000001ff0000000800380000000000000000000000000000385cf8ffffff10cdf7bb2cd063000000000000042d343507fffffffffff800404220fffffff800000040420f0000", &mut a);
+        extend_vec_from_hex("ffffff0000010080800000000000380000001adfc7040000000000000000000007fffffffffffffe1000000000000000000001555600000000000000000000000000000000000000000000000000000000000000000000ffffffffffffff0300000000000000000000000000000000000000000000000000008000000000000000ffffffffffffff7f00000000000000ffffff151515111515155555555555d6eeffffff00", &mut a);
         super::do_test(&a);
     }
 }

fuzz/fuzz_targets/decode_program.rs

@@ -6,25 +6,21 @@ use simplicity::jet::Core;
 use simplicity::{BitIter, BitWriter, RedeemNode};
 
 fn do_test(data: &[u8]) {
-    let mut iter = BitIter::new(data.iter().cloned());
-
-    if let Ok(program) = RedeemNode::<Core>::decode(&mut iter) {
-        let bit_len = iter.n_total_read();
-
-        let mut sink = Vec::<u8>::new();
-        let mut w = BitWriter::from(&mut sink);
-        program.encode(&mut w).expect("encoding to vector");
-        w.flush_all().expect("flushing");
-        assert_eq!(w.n_total_written(), bit_len);
-
-        // RedeemNode::<Value, Core>::decode() may stop reading `data` mid-byte:
-        // copy trailing bits from `data` to `sink`
-        if bit_len % 8 != 0 {
-            let mask = !(1u8 << (8 - (bit_len % 8)));
-            let idx = sink.len() - 1;
-            sink[idx] |= data[idx] & mask;
-        }
-        assert_eq!(sink, &data[0..sink.len()]);
+    let prog_iter = BitIter::new(data.iter().cloned());
+    let wit_iter = BitIter::new(core::iter::repeat(0));
+    if let Ok(program) = RedeemNode::<Core>::decode(prog_iter, wit_iter) {
+        let mut prog_reser = Vec::<u8>::new();
+        let mut wit_reser = std::io::sink();
+
+        let mut prog_w = BitWriter::from(&mut prog_reser);
+        let mut wit_w = BitWriter::from(&mut wit_reser);
+        program
+            .encode(&mut prog_w, &mut wit_w)
+            .expect("encoding to vector");
+        prog_w.flush_all().expect("flushing");
+        wit_w.flush_all().expect("flushing");
+
+        assert_eq!(prog_reser, &data[0..prog_reser.len()]);
     }
 }
 

simpcli/src/main.rs

@@ -151,9 +151,9 @@ fn main() -> Result<(), String> {
         Command::Disassemble => {
             let v = base64::Engine::decode(&STANDARD, first_arg.as_bytes())
                 .map_err(|e| format!("failed to parse base64: {}", e))?;
-            let mut iter = BitIter::from(v.into_iter());
-            let commit = CommitNode::decode(&mut iter)
-                .map_err(|e| format!("failed to decode program: {}", e))?;
+            let iter = BitIter::from(v.into_iter());
+            let commit =
+                CommitNode::decode(iter).map_err(|e| format!("failed to decode program: {}", e))?;
             let prog = Forest::<DefaultJet>::from_program(commit);
             println!("{}", prog.string_serialize());
         }

simplicity-sys/src/tests/mod.rs

@@ -285,6 +285,7 @@ pub struct TestData {
     pub amr: [u8; 32],
     pub imr: [u8; 32],
     pub prog: Vec<u8>,
+    pub witness: Vec<u8>,
     pub cost: ubounded,
 }
 
@@ -299,6 +300,11 @@ mod test_data {
                 amr: parse_root(&ffi::schnorr0_amr),
                 imr: parse_root(&ffi::schnorr0_imr),
                 prog: slice::from_raw_parts(ffi::schnorr0.as_ptr(), ffi::sizeof_schnorr0).into(),
+                witness: slice::from_raw_parts(
+                    ffi::schnorr0_witness.as_ptr(),
+                    ffi::sizeof_schnorr0_witness,
+                )
+                .into(),
                 cost: ffi::schnorr0_cost,
             }
         }
@@ -311,6 +317,11 @@ mod test_data {
                 amr: parse_root(&ffi::schnorr6_amr),
                 imr: parse_root(&ffi::schnorr6_imr),
                 prog: slice::from_raw_parts(ffi::schnorr6.as_ptr(), ffi::sizeof_schnorr6).into(),
+                witness: slice::from_raw_parts(
+                    ffi::schnorr6_witness.as_ptr(),
+                    ffi::sizeof_schnorr6_witness,
+                )
+                .into(),
                 cost: ffi::schnorr6_cost,
             }
         }
@@ -324,6 +335,11 @@ mod test_data {
                 imr: parse_root(&ffi::ctx8Pruned_imr),
                 prog: slice::from_raw_parts(ffi::ctx8Pruned.as_ptr(), ffi::sizeof_ctx8Pruned)
                     .into(),
+                witness: slice::from_raw_parts(
+                    ffi::ctx8Pruned_witness.as_ptr(),
+                    ffi::sizeof_ctx8Pruned_witness,
+                )
+                .into(),
                 cost: ffi::ctx8Pruned_cost,
             }
         }
@@ -337,6 +353,11 @@ mod test_data {
                 imr: parse_root(&ffi::ctx8Unpruned_imr),
                 prog: slice::from_raw_parts(ffi::ctx8Unpruned.as_ptr(), ffi::sizeof_ctx8Unpruned)
                     .into(),
+                witness: slice::from_raw_parts(
+                    ffi::ctx8Unpruned_witness.as_ptr(),
+                    ffi::sizeof_ctx8Unpruned_witness,
+                )
+                .into(),
                 cost: ffi::ctx8Unpruned_cost,
             }
         }

src/bit_encoding/decode.rs

@@ -416,10 +416,10 @@ mod tests {
         decode_expression::<_, Core>(&mut iter).unwrap();
         // ...but NOT as a CommitNode
         let iter = BitIter::from(&justjet[..]);
-        CommitNode::<Core>::decode::<_>(iter).unwrap_err();
+        CommitNode::<Core>::decode(iter).unwrap_err();
         // ...or as a RedeemNode
         let iter = BitIter::from(&justjet[..]);
-        RedeemNode::<Core>::decode::<_>(iter).unwrap_err();
+        RedeemNode::<Core>::decode(iter, BitIter::from(&[][..])).unwrap_err();
     }
 
     #[test]

src/bit_encoding/encode.rs

@@ -278,25 +278,11 @@ pub fn encode_witness<'a, W: io::Write, I>(witness: I, w: &mut BitWriter<W>) ->
 where
     I: Iterator<Item = &'a Value> + Clone,
 {
-    let mut bit_len = 0;
-    let n_start = w.n_total_written();
-
-    for value in witness.clone() {
-        bit_len += value.len();
+    let mut len = 0;
+    for value in witness {
+        len += encode_value(value, w)?;
     }
-
-    if bit_len == 0 {
-        w.write_bit(false)?;
-    } else {
-        w.write_bit(true)?;
-        encode_natural(bit_len, w)?;
-
-        for value in witness {
-            encode_value(value, w)?;
-        }
-    }
-
-    Ok(w.n_total_written() - n_start)
+    Ok(len)
 }
 
 /// Encode a value to bits.

src/bit_machine/mod.rs

@@ -542,14 +542,16 @@ mod tests {
     #[cfg(feature = "elements")]
     fn run_program_elements(
         prog_bytes: &[u8],
+        witness_bytes: &[u8],
         cmr_str: &str,
         amr_str: &str,
         imr_str: &str,
     ) -> Result<Arc<Value>, ExecutionError> {
         let prog_hex = prog_bytes.as_hex();
 
-        let iter = BitIter::from(prog_bytes);
-        let prog = match RedeemNode::<Elements>::decode(iter) {
+        let prog = BitIter::from(prog_bytes);
+        let witness = BitIter::from(witness_bytes);
+        let prog = match RedeemNode::<Elements>::decode(prog, witness) {
             Ok(prog) => prog,
             Err(e) => panic!("program {} failed: {}", prog_hex, e),
         };
@@ -605,6 +607,7 @@ mod tests {
 
         let res = run_program_elements(
             &[0xcf, 0xe1, 0x8f, 0xb4, 0x40, 0x28, 0x87, 0x04, 0x00],
+            &[],
             "ec48102095c13fcdc1d539de2848ae287acdea249e2cda6f0d8f34bccd292294",
             "abd217b5ea14d7da249a03e16dd047b136a2efec4b82c1b60675297d782b51ad",
             "dea130f31a0754ea2f82ad570f7a4882c9e465b6bdd6f8be4d6d68342a57dff3",

src/lib.rs

@@ -86,8 +86,6 @@ pub enum Error {
     NoMoreWitnesses,
     /// Finalization failed; did not have enough witness data to satisfy program.
     IncompleteFinalization,
-    /// Witness has different length than defined in its preamble
-    InconsistentWitnessLength,
     /// Tried to parse a jet but the name wasn't recognized
     InvalidJetName(String),
     /// Policy error
@@ -107,9 +105,6 @@ impl fmt::Display for Error {
             }
             Error::Type(ref e) => fmt::Display::fmt(e, f),
             Error::IncompleteFinalization => f.write_str("unable to satisfy program"),
-            Error::InconsistentWitnessLength => {
-                f.write_str("witness has different length than defined in its preamble")
-            }
             Error::InvalidJetName(s) => write!(f, "unknown jet `{}`", s),
             Error::NoMoreWitnesses => f.write_str("no more witness data available"),
             #[cfg(feature = "elements")]
@@ -127,7 +122,6 @@ impl std::error::Error for Error {
             Error::Type(ref e) => Some(e),
             Error::NoMoreWitnesses => None,
             Error::IncompleteFinalization => None,
-            Error::InconsistentWitnessLength => None,
             Error::InvalidJetName(..) => None,
             #[cfg(feature = "elements")]
             Error::Policy(ref e) => Some(e),

src/node/commit.rs

@@ -281,13 +281,13 @@ impl<J: Jet> CommitNode<J> {
 
     /// Encode a Simplicity program to a vector of bytes, without any witness data.
     pub fn encode_to_vec(&self) -> Vec<u8> {
-        let mut program_and_witness_bytes = Vec::<u8>::new();
-        let mut writer = BitWriter::new(&mut program_and_witness_bytes);
+        let mut program = Vec::<u8>::new();
+        let mut writer = BitWriter::new(&mut program);
         self.encode(&mut writer)
             .expect("write to vector never fails");
-        debug_assert!(!program_and_witness_bytes.is_empty());
+        debug_assert!(!program.is_empty());
 
-        program_and_witness_bytes
+        program
     }
 }
 

src/node/mod.rs

@@ -679,8 +679,10 @@ mod tests {
 
     fn check_merkle_roots(test: &TestData) {
         let prog = BitIter::from(test.prog.as_slice());
-        ffi::tests::run_program(&test.prog, ffi::tests::TestUpTo::CheckOneOne).unwrap();
-        let prog = RedeemNode::<Elements>::decode(prog).unwrap();
+        let witness = BitIter::from(test.witness.as_slice());
+        ffi::tests::run_program(&test.prog, &test.witness, ffi::tests::TestUpTo::CheckOneOne)
+            .unwrap();
+        let prog = RedeemNode::<Elements>::decode(prog, witness).unwrap();
         assert_eq!(prog.cmr().to_byte_array(), test.cmr);
         assert_eq!(prog.amr().to_byte_array(), test.amr);
         assert_eq!(prog.imr().to_byte_array(), test.imr);