From 334b9133e72de3ec1e7ef87e2f55110dbe181b2f Mon Sep 17 00:00:00 2001 From: David Pearce Date: Wed, 16 Oct 2024 09:15:29 +1300 Subject: [PATCH] feat: setting column size factor manually (#274) --- src/column.rs | 17 ++- src/compiler/generator.rs | 28 +++-- src/compiler/mod.rs | 2 + src/compiler/node.rs | 3 + src/compiler/parser/definitions.rs | 2 + src/compiler/parser/mod.rs | 2 + src/compiler/parser/parser.rs | 19 +++ src/import.rs | 183 ++++++++++++++++++++--------- tests/len_multiplier_1.accepts | 2 + tests/len_multiplier_1.lisp | 3 + tests/len_multiplier_1.rejects | 2 + tests/len_multiplier_2.lisp | 5 + 12 files changed, 201 insertions(+), 67 deletions(-) create mode 100644 tests/len_multiplier_1.accepts create mode 100644 tests/len_multiplier_1.lisp create mode 100644 tests/len_multiplier_1.rejects create mode 100644 tests/len_multiplier_2.lisp diff --git a/src/column.rs b/src/column.rs index 01873a5..376353c 100644 --- a/src/column.rs +++ b/src/column.rs @@ -793,6 +793,7 @@ impl<'a> Iterator for ValueBackingIter<'a> { pub struct Register { pub handle: Option, pub magma: Magma, + pub length_multiplier: usize, #[serde(skip_serializing, skip_deserializing, default)] backing: Option, width: usize, @@ -1163,10 +1164,16 @@ impl ColumnSet { self._cols.iter() } - pub(crate) fn new_register(&mut self, handle: Handle, magma: Magma) -> RegisterID { + pub(crate) fn new_register( + &mut self, + handle: Handle, + magma: Magma, + length_multiplier: usize, + ) -> RegisterID { self.registers.push(Register { handle: Some(handle), magma, + length_multiplier, backing: None, width: crate::constants::col_count_magma(magma), }); @@ -1219,7 +1226,9 @@ impl ColumnSet { } pub fn insert_column_and_register(&mut self, mut column: Column) -> Result { - column.register = Some(self.new_register(column.handle.clone(), column.t)); + let length_multiplier = column.intrinsic_size_factor.unwrap_or(1); + column.register = + Some(self.new_register(column.handle.clone(), column.t, length_multiplier)); self.insert_column(column) } @@ -1227,7 +1236,9 @@ impl ColumnSet { if self.cols.contains_key(&column.handle) { None } else { - column.register = Some(self.new_register(column.handle.clone(), column.t)); + let length_multiplier = column.intrinsic_size_factor.unwrap_or(1); + column.register = + Some(self.new_register(column.handle.clone(), column.t, length_multiplier)); self.maybe_insert_column(column) } } diff --git a/src/compiler/generator.rs b/src/compiler/generator.rs index fd65de6..0aed499 100644 --- a/src/compiler/generator.rs +++ b/src/compiler/generator.rs @@ -434,9 +434,11 @@ impl ConstraintSet { // module-global columns all have their own register for c in pool.root { + let col = self.columns.column(&c).unwrap(); let reg = self.columns.new_register( self.handle(&c).to_owned(), - self.columns.column(&c).unwrap().t, + col.t, + col.intrinsic_size_factor.unwrap_or(1), ); self.columns.assign_register(&c, reg).unwrap(); } @@ -459,9 +461,9 @@ impl ConstraintSet { .filter_map(|v| v.get(i)) .map(|r| self.handle(r).name.to_owned()) .join("_xor_"); - let reg = self - .columns - .new_register(Handle::new(module, names), *magma); + let reg = + self.columns + .new_register(Handle::new(module, names), *magma, *size); for cols in sets.values() { if let Some(col_id) = cols.get(i) { self.columns.assign_register(col_id, reg).unwrap(); @@ -494,7 +496,11 @@ impl ConstraintSet { | Computation::CyclicFrom { target, .. } | Computation::Composite { target, .. } => { let col = self.columns.column(&target).unwrap(); - let reg = self.columns.new_register(col.handle.clone(), col.t); + let reg = self.columns.new_register( + col.handle.clone(), + col.t, + col.intrinsic_size_factor.unwrap_or(1), + ); self.columns.assign_register(&target, reg).unwrap(); } Computation::Sorted { froms, tos, .. } => { @@ -504,7 +510,11 @@ impl ConstraintSet { .entry(self.columns.column(f).unwrap().register.unwrap()) .or_insert_with(|| { let col = self.columns.column(t).unwrap(); - self.columns.new_register(col.handle.clone(), col.t) + self.columns.new_register( + col.handle.clone(), + col.t, + col.intrinsic_size_factor.unwrap_or(1), + ) }); self.columns.assign_register(t, *reg).unwrap(); } @@ -522,7 +532,11 @@ impl ConstraintSet { .chain(delta_bytes.iter()) { let col = self.columns.column(r).unwrap(); - let reg = self.columns.new_register(col.handle.clone(), col.t); + let reg = self.columns.new_register( + col.handle.clone(), + col.t, + col.intrinsic_size_factor.unwrap_or(1), + ); self.columns.assign_register(r, reg).unwrap(); } } diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs index dd7b6ea..c4a3ab8 100644 --- a/src/compiler/mod.rs +++ b/src/compiler/mod.rs @@ -70,11 +70,13 @@ pub fn make, S2: AsRef>( padding_value, base, must_prove, + length_multiplier, .. } => { let column = Column::builder() .handle(handle.as_handle().clone()) .and_padding_value(padding_value.to_owned()) + .and_intrinsic_size_factor(length_multiplier.to_owned()) .kind(k.to_nil()) .t(symbol.t().m()) .must_prove(*must_prove) diff --git a/src/compiler/node.rs b/src/compiler/node.rs index 218da6b..3d2cbf4 100644 --- a/src/compiler/node.rs +++ b/src/compiler/node.rs @@ -212,6 +212,7 @@ pub enum Expression { kind: Kind>, must_prove: bool, padding_value: Option, + length_multiplier: Option, base: Base, }, ArrayColumn { @@ -304,6 +305,7 @@ impl Node { base: Option, kind: Option>>, padding_value: Option, + length_multiplier: Option, must_prove: Option, t: Option, ) -> Node { @@ -327,6 +329,7 @@ impl Node { kind: kind.unwrap_or(Kind::Computed), must_prove: must_prove.unwrap_or(false), padding_value, + length_multiplier, base: base.unwrap_or_else(|| t.unwrap_or(Magma::native()).into()), }, _t: Some(Type::Column(t.unwrap_or(Magma::native()))), diff --git a/src/compiler/parser/definitions.rs b/src/compiler/parser/definitions.rs index d8d889a..7905da8 100644 --- a/src/compiler/parser/definitions.rs +++ b/src/compiler/parser/definitions.rs @@ -47,6 +47,7 @@ fn reduce(e: &AstNode, ctx: &mut Scope, settings: &CompileSettings) -> Result<() t, kind, padding_value, + length_multiplier, must_prove, base, } => { @@ -63,6 +64,7 @@ fn reduce(e: &AstNode, ctx: &mut Scope, settings: &CompileSettings) -> Result<() Kind::Expression(_) => Kind::Computed, }) .and_padding_value(*padding_value) + .and_length_multiplier(*length_multiplier) .t(t.m()) .must_prove(*must_prove) .base(*base) diff --git a/src/compiler/parser/mod.rs b/src/compiler/parser/mod.rs index 71008f8..7e93193 100644 --- a/src/compiler/parser/mod.rs +++ b/src/compiler/parser/mod.rs @@ -172,6 +172,8 @@ pub enum Token { kind: Kind>, /// the value to pad the column with; defaults to 0 if None padding_value: Option, + /// the length multiplier for this column; defaults to 1 if None + length_multiplier: Option, /// if set, generate constraint to prove the column type must_prove: bool, /// which numeric base should be used to display column values; this is a purely aesthetic setting diff --git a/src/compiler/parser/parser.rs b/src/compiler/parser/parser.rs index 4aac849..ea27900 100755 --- a/src/compiler/parser/parser.rs +++ b/src/compiler/parser/parser.rs @@ -188,6 +188,7 @@ struct ColumnAttributes { must_prove: bool, range: OnceCell>>, padding_value: OnceCell, + length_multiplier: OnceCell, base: OnceCell, computation: Option, } @@ -220,6 +221,7 @@ fn parse_column_attributes(source: AstNode) -> Result { Array, Computation, PaddingValue, + LengthMultiplier, Base, } let re_type = regex_lite::Regex::new( @@ -257,6 +259,8 @@ fn parse_column_attributes(source: AstNode) -> Result { ":padding" => ColumnParser::PaddingValue, // how to display the column values in debug ":display" => ColumnParser::Base, + // a specific length multiplier + ":length" => ColumnParser::LengthMultiplier, _ => { if let Some(caps) = re_type.captures(kw) { let raw_magma = if let Some(integer) = caps.name("Integer") { @@ -339,6 +343,19 @@ fn parse_column_attributes(source: AstNode) -> Result { })?; ColumnParser::Begin } + ColumnParser::LengthMultiplier => { + attributes + .length_multiplier + .set(x.as_u64()? as usize) + .map_err(|_| { + anyhow!( + "invalid length multiplier for column {} ({})", + attributes.name, + x.as_i64().unwrap() + ) + })?; + ColumnParser::Begin + } ColumnParser::Base => { let base = if let Token::Keyword(ref kw) = x.class { kw.as_str().try_into()? @@ -363,6 +380,7 @@ fn parse_column_attributes(source: AstNode) -> Result { ColumnParser::Array => bail!("incomplete :array definition"), ColumnParser::Computation => bail!("incomplate :comp definition"), ColumnParser::PaddingValue => bail!("incomplete :padding definition"), + ColumnParser::LengthMultiplier => bail!("incomplete :length definition"), ColumnParser::Base => bail!("incomplete :display definition"), } Ok(attributes) @@ -411,6 +429,7 @@ fn parse_defcolumns>>( .map(|c| Kind::Expression(Box::new(c))) .unwrap_or(Kind::Commitment), padding_value: column_attributes.padding_value.get().cloned(), + length_multiplier: column_attributes.length_multiplier.get().cloned(), must_prove: column_attributes.must_prove, base, } diff --git a/src/import.rs b/src/import.rs index fa184e3..16d3715 100644 --- a/src/import.rs +++ b/src/import.rs @@ -159,29 +159,38 @@ pub fn parse_binary_trace(tracefile: &str, cs: &mut ConstraintSet, keep_raw: boo let register_bytes = trace_reader .slice(trace_register.length as usize * trace_register.bytes_per_element)?; - if let Some(Register { magma, .. }) = cs.columns.register(&column_ref) { - let mut xs = (if keep_raw { 0 } else { -1 }..trace_register.length) + if let Some(Register { + magma, + length_multiplier, + .. + }) = cs.columns.register(&column_ref) + { + let mut xs = Vec::new(); + if !keep_raw { + // Add initial padding row + for _i in 0..*length_multiplier { + xs.push(CValue::zero()); + } + } + // Read data + let mut rs = (0..trace_register.length) .into_par_iter() .map(|i| { - if i == -1 { - Ok(CValue::zero()) - } else { - let i = i as usize; - register_bytes - .get( - i * trace_register.bytes_per_element - ..(i + 1) * trace_register.bytes_per_element, - ) - .ok_or_else(|| anyhow!("error reading {}th element", i)) - .and_then(|bs| { - CValue::try_from(BigInt::from_bytes_be(Sign::Plus, bs)) - .with_context(|| anyhow!("while parsing {}th element", i)) - .and_then(|x| magma.rm().validate(x)) - }) - .with_context(|| anyhow!("reading {}th element", i)) - } + let i = i as usize; + register_bytes + .get( + i * trace_register.bytes_per_element + ..(i + 1) * trace_register.bytes_per_element, + ) + .ok_or_else(|| anyhow!("error reading {}th element", i)) + .and_then(|bs| { + CValue::try_from(BigInt::from_bytes_be(Sign::Plus, bs)) + .with_context(|| anyhow!("while parsing {}th element", i)) + .and_then(|x| magma.rm().validate(x)) + }) + .with_context(|| anyhow!("reading {}th element", i)) }) - .collect::>>() + .collect::>>() .with_context(|| { anyhow!( "reading data for {} ({} elts. expected)", @@ -189,7 +198,20 @@ pub fn parse_binary_trace(tracefile: &str, cs: &mut ConstraintSet, keep_raw: boo trace_register.length ) })?; - + // Append values + xs.extend(rs); + // Sanity check length has multiplier as factor + if xs.len() % length_multiplier != 0 { + bail!( + "{} has an incorrect length multiplier: length {} not divisible by {}", + trace_register.handle.to_string().blue(), + xs.len(), + length_multiplier, + ); + } + // Extract module-normalised length + let xs_len = xs.len() / length_multiplier; + // let module_min_len = cs .columns .min_len @@ -207,23 +229,22 @@ pub fn parse_binary_trace(tracefile: &str, cs: &mut ConstraintSet, keep_raw: boo // required. // Atomic columns are always padded with zeroes, so there is // no need to trigger a more complex padding system. - if !keep_raw && xs.len() < module_min_len { + if !keep_raw && xs_len < module_min_len { xs.reverse(); - xs.resize(module_min_len, CValue::zero()); // TODO: register padding values + xs.resize(module_min_len * length_multiplier, CValue::zero()); // TODO: register padding values xs.reverse(); } let module_raw_size = - cs.effective_len_or_set(&trace_register.handle.module, xs.len() as isize); - if xs.len() as isize != module_raw_size { + cs.effective_len_or_set(&trace_register.handle.module, xs_len as isize); + if xs_len as isize != module_raw_size { bail!( "{} has an incorrect length: expected {}, found {}", trace_register.handle.to_string().blue(), module_raw_size.to_string().red().bold(), - xs.len().to_string().yellow().bold(), + xs_len.to_string().yellow().bold(), ); } - cs.columns .set_register_value(&trace_register.handle.into(), xs, module_spilling)? } else { @@ -303,14 +324,23 @@ pub fn read_trace_str(tracestr: &[u8], cs: &mut ConstraintSet, keep_raw: bool) - } #[cfg(not(all(target_arch = "x86_64", target_feature = "avx")))] -fn parse_column(xs: &[Value], h: &Handle, t: Magma, keep_raw: bool) -> Result> { +fn parse_column( + xs: &[Value], + h: &Handle, + t: Magma, + keep_raw: bool, + length_multiplier: usize, +) -> Result> { let mut cache_num = cached::SizedCache::with_size(200000); // ~1.60MB cache let mut cache_str = cached::SizedCache::with_size(200000); // ~1.60MB cache - let mut r = if keep_raw { - Vec::new() - } else { - vec![CValue::zero()] - }; + let mut r = Vec::new(); + + if !keep_raw { + // Add initial padding row + for _i in 0..length_multiplier { + r.push(CValue::zero()); + } + } let xs = xs .iter() .map(|x| match x { @@ -336,13 +366,20 @@ fn parse_column(xs: &[Value], h: &Handle, t: Magma, keep_raw: bool) -> Result Result> { +fn parse_column( + xs: &[Value], + h: &Handle, + t: Magma, + keep_raw: bool, + length_multiplier: usize, +) -> Result> { let mut cache = cached::SizedCache::with_size(200000); // ~1.60MB cache - let mut r = if keep_raw { - Vec::new() - } else { - vec![CValue::zero()] - }; + if !keep_raw { + // Add initial padding row + for i in 0..length_multiplier { + r.push(CValue::zero()); + } + } let xs = xs .iter() .map(|x| { @@ -405,9 +442,14 @@ pub fn fill_traces_from_json( let module_spilling = cs.spilling_for_column(&handle); if let Result::Ok(Column { - t, padding_value, .. + t, + padding_value, + intrinsic_size_factor, + .. }) = cs.columns.column(&handle) { + // Determing length multiplier (if none, then default to 1) + let length_multiplier = intrinsic_size_factor.unwrap_or(1); trace!("inserting {} ({})", handle, xs.len()); if let Some(first_column) = initiator.as_mut() { if first_column.is_empty() { @@ -418,67 +460,94 @@ pub fn fill_traces_from_json( let module_spilling = module_spilling .ok_or_else(|| anyhow!("no spilling found for {}", handle.pretty()))?; - let mut xs = parse_column(xs, handle.as_handle(), *t, keep_raw) - .with_context(|| anyhow!("importing {}", handle.pretty()))?; + let mut xs = + parse_column(xs, handle.as_handle(), *t, keep_raw, length_multiplier) + .with_context(|| anyhow!("importing {}", handle.pretty()))?; + // Sanity check length has multiplier as factor + if xs.len() % length_multiplier != 0 { + bail!( + "{} has an incorrect length multiplier: length {} not divisible by {}", + handle.to_string().blue(), + xs.len(), + length_multiplier, + ); + } // If the parsed column is not long enought w.r.t. the // minimal module length, prepend it with as many zeroes as // required. // Atomic columns are always padded with zeroes, so there is // no need to trigger a more complex padding system. - if !keep_raw && xs.len() < module_min_len { + if !keep_raw && xs.len() < module_min_len * length_multiplier { trace!( "padding {} to min module length ({} => {})", handle, - xs.len(), + xs.len() * length_multiplier, module_min_len ); xs.reverse(); - xs.resize_with(module_min_len, || { + xs.resize_with(module_min_len * length_multiplier, || { padding_value.clone().unwrap_or_default() }); xs.reverse(); } - + let xs_len = xs.len() / length_multiplier; // The first column sets the size of its module - let module_raw_size = cs.effective_len_or_set(&module, xs.len() as isize); - if xs.len() as isize != module_raw_size { + let module_raw_size = cs.effective_len_or_set(&module, xs_len as isize); + if xs_len as isize != module_raw_size { bail!( "{} has an incorrect length: expected {} (from {}), found {}", handle.to_string().blue(), module_raw_size.to_string().red().bold(), initiator.as_ref().unwrap(), - xs.len().to_string().yellow().bold(), + xs_len.to_string().yellow().bold(), ); } cs.columns.set_column_value(&handle, xs, module_spilling)? - } else if let Some(Register { magma, .. }) = cs.columns.register(&handle) { + } else if let Some(Register { + magma, + length_multiplier, + .. + }) = cs.columns.register(&handle) + { let module_spilling = module_spilling .ok_or_else(|| anyhow!("no spilling found for {}", handle.pretty()))?; - let mut xs = parse_column(xs, handle.as_handle(), *magma, keep_raw) - .with_context(|| anyhow!("importing {}", handle.pretty()))?; + let mut xs = + parse_column(xs, handle.as_handle(), *magma, keep_raw, *length_multiplier) + .with_context(|| anyhow!("importing {}", handle.pretty()))?; + // Sanity check length has multiplier as factor + if xs.len() % length_multiplier != 0 { + bail!( + "{} has an incorrect length multiplier: length {} not divisible by {}", + handle.to_string().blue(), + xs.len(), + length_multiplier, + ); + } + // Extract module-normalised length + let xs_len = xs.len() / length_multiplier; // If the parsed column is not long enought w.r.t. the // minimal module length, prepend it with as many zeroes as // required. // Atomic columns are always padded with zeroes, so there is // no need to trigger a more complex padding system. - if xs.len() < module_min_len { + if !keep_raw && xs_len < module_min_len { xs.reverse(); - xs.resize(module_min_len, CValue::zero()); // TODO: register padding values + xs.resize(module_min_len * length_multiplier, CValue::zero()); // TODO: register padding values xs.reverse(); } - let module_raw_size = cs.effective_len_or_set(&module, xs.len() as isize); - if xs.len() as isize != module_raw_size { + let module_raw_size = cs.effective_len_or_set(&module, xs_len as isize); + if xs_len as isize != module_raw_size { bail!( "{} has an incorrect length: expected {} (from {}), found {}", handle.to_string().blue(), module_raw_size.to_string().red().bold(), initiator.as_ref().unwrap(), - xs.len().to_string().yellow().bold(), + xs_len.to_string().yellow().bold(), ); } diff --git a/tests/len_multiplier_1.accepts b/tests/len_multiplier_1.accepts new file mode 100644 index 0000000..e4d92a1 --- /dev/null +++ b/tests/len_multiplier_1.accepts @@ -0,0 +1,2 @@ +{ "": { "X": [ 1 ], "Y": [ 1 ], "A": [ 1, 2 ] } } +{ "": { "X": [ 1 ], "Y": [ 1 ], "A": [ 1, 2, 3 ] } } diff --git a/tests/len_multiplier_1.lisp b/tests/len_multiplier_1.lisp new file mode 100644 index 0000000..9ee937c --- /dev/null +++ b/tests/len_multiplier_1.lisp @@ -0,0 +1,3 @@ +(defcolumns X Y (A :length 2)) +(definterleaved (Z) (X Y)) +(defconstraint tmp () (vanishes! (- A Z))) diff --git a/tests/len_multiplier_1.rejects b/tests/len_multiplier_1.rejects new file mode 100644 index 0000000..5b2f189 --- /dev/null +++ b/tests/len_multiplier_1.rejects @@ -0,0 +1,2 @@ +{ "": { "X": [ 1 ], "Y": [ 1 ], "A": [ 1, 2 ] } } +{ "": { "X": [ 1 ], "Y": [ 1 ], "A": [ 1, 2, 3 ] } } diff --git a/tests/len_multiplier_2.lisp b/tests/len_multiplier_2.lisp new file mode 100644 index 0000000..64d8d81 --- /dev/null +++ b/tests/len_multiplier_2.lisp @@ -0,0 +1,5 @@ +(defcolumns (A :length 2) (P :binary@prove :length 2)) +(defperspective perp1 P (X Y)) +(definterleaved (Z) (perp1/X perp1/Y)) + +(defconstraint c1 (:perspective perp1) (vanishes! (- A Z)))