From 021f9d29da9980604fdf434028e613c852c3745e Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 20 May 2024 13:13:34 +0200 Subject: [PATCH 01/93] add Int64 check method which doesn't allow ambiguous 0, to be enabled in v2 --- src/lib/provable/int.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index 43f75c4cdb..aa782faeb2 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -969,7 +969,7 @@ class Sign extends CircuitValue { } static check(x: Sign) { // x^2 === 1 <=> x === 1 or x === -1 - x.value.square().assertEquals(Field(1)); + x.value.square().assertEquals(1); } static empty(): InstanceType { return Sign.one as any; @@ -994,7 +994,7 @@ class Sign extends CircuitValue { return new Sign(this.value.mul(y.value)); } isPositive() { - return this.value.equals(Field(1)); + return this.value.equals(1); } toString() { return this.value.toString(); @@ -1019,7 +1019,7 @@ type BalanceChange = Types.AccountUpdate['body']['balanceChange']; */ class Int64 extends CircuitValue implements BalanceChange { // * in the range [-2^64+1, 2^64-1], unlike a normal int64 - // * under- and overflowing is disallowed, similar to UInt64, unlike a normal int64+ + // * under- and overflowing is disallowed, similar to UInt64, unlike a normal int64 @prop magnitude: UInt64; // absolute value @prop sgn: Sign; // +/- 1 @@ -1060,9 +1060,9 @@ class Int64 extends CircuitValue implements BalanceChange { let isValidNegative = Field.ORDER - xBigInt < TWO64; // {-2^64 + 1,...,-1} if (!isValidPositive && !isValidNegative) throw Error(`Int64: Expected a value between (-2^64, 2^64), got ${x}`); - let magnitude = Field(isValidPositive ? x.toString() : x.neg().toString()); + let magnitude = (isValidPositive ? x : x.neg()).toConstant(); let sign = isValidPositive ? Sign.one : Sign.minusOne; - return new Int64(new UInt64(magnitude.value), sign); + return new Int64(UInt64.Unsafe.fromField(magnitude), sign); } // this doesn't check ranges because we assume they're already checked on UInts @@ -1220,6 +1220,20 @@ class Int64 extends CircuitValue implements BalanceChange { isPositive() { return this.sgn.isPositive(); } + + // TODO enable this check method in v2 + static checkV2({ magnitude, sgn }: { magnitude: UInt64; sgn: Sign }) { + // check that the magnitude is in range + UInt64.check(magnitude); + // check that the sign is valid + Sign.check(sgn); + + // check unique representation of 0: we can't have magnitude = 0 and sgn = -1 + // magnitude + sign != -1 (this check works because magnitude >= 0) + magnitude.value + .add(sgn.value) + .assertNotEquals(-1, 'Int64: 0 must have positive sign'); + } } /** From 8f40313adcabe4f3ae9a2478f8b2e8bde1cf15e9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 20 May 2024 13:16:24 +0200 Subject: [PATCH 02/93] fix `neg()` in the unique representation --- src/lib/provable/int.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index aa782faeb2..00bbbc3b33 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -1142,14 +1142,26 @@ class Int64 extends CircuitValue implements BalanceChange { } /** - * Negates the value. - * - * `Int64.from(5).neg()` will turn into `Int64.from(-5)` + * @deprecated Use {@link negV2()} instead. */ neg() { // doesn't need further check if `this` is valid return new Int64(this.magnitude, this.sgn.neg()); } + + /** + * Negates the value. + * + * `Int64.from(5).neg()` will turn into `Int64.from(-5)` + */ + negV2() { + return Provable.if( + this.magnitude.value.equals(0), + Int64.zero, + new Int64(this.magnitude, this.sgn.neg()) + ); + } + /** * Addition with overflow checking. */ From 85ea26387cfa7b1fbc1fba111a775447699a6ca8 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 20 May 2024 13:21:22 +0200 Subject: [PATCH 03/93] deprecate misleading `isPositive()` and replace with to v2 methods --- src/lib/provable/int.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index 00bbbc3b33..4acd93420c 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -1227,12 +1227,27 @@ class Int64 extends CircuitValue implements BalanceChange { this.toField().assertEquals(y_.toField(), message); } /** - * Checks if the value is positive. + * @deprecated Use {@link isPositiveV2()} or {@link isNonNegativeV2()} instead. */ isPositive() { return this.sgn.isPositive(); } + /** + * Checks if the value is positive (x > 0). + */ + isPositiveV2() { + return this.magnitude.equals(UInt64.zero).not().and(this.sgn.isPositive()); + } + + /** + * Checks if the value is non-negative (x >= 0). + */ + isNonNegativeV2() { + // this will be the correct logic when `checkV2` is enabled + return this.sgn.isPositive(); + } + // TODO enable this check method in v2 static checkV2({ magnitude, sgn }: { magnitude: UInt64; sgn: Sign }) { // check that the magnitude is in range From c919ab883176d02e35dae423673063e35dcec5ca Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 20 May 2024 13:57:34 +0200 Subject: [PATCH 04/93] fixed version of Int64.mod() --- src/lib/provable/int.ts | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index 4acd93420c..85628d8b63 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -996,6 +996,10 @@ class Sign extends CircuitValue { isPositive() { return this.value.equals(1); } + isNegative() { + return this.value.equals(-1); + } + toString() { return this.value.toString(); } @@ -1143,6 +1147,7 @@ class Int64 extends CircuitValue implements BalanceChange { /** * @deprecated Use {@link negV2()} instead. + * The current implementation will not be backwards-compatible with v2. */ neg() { // doesn't need further check if `this` is valid @@ -1188,7 +1193,7 @@ class Int64 extends CircuitValue implements BalanceChange { * * `x.div(y)` returns the floor of `x / y`, that is, the greatest * `z` such that `z * y <= x`. - * + * On negative numbers, this rounds towards zero. */ div(y: Int64 | number | string | bigint | UInt64 | UInt32) { let y_ = Int64.from(y); @@ -1196,16 +1201,29 @@ class Int64 extends CircuitValue implements BalanceChange { let sign = this.sgn.mul(y_.sgn); return new Int64(quotient, sign); } + + /** + * @deprecated Use {@link modV2()} instead. + * This implementation is vulnerable whenever `this` is zero. + * It allows the prover to return `y` instead of 0 as the result. + */ + mod(y: UInt64 | number | string | bigint | UInt32) { + let y_ = UInt64.from(y); + let rest = this.magnitude.divMod(y_).rest.value; + rest = Provable.if(this.isPositive(), rest, y_.value.sub(rest)); + return new Int64(new UInt64(rest.value)); + } + /** * Integer remainder. * * `x.mod(y)` returns the value `z` such that `0 <= z < y` and * `x - z` is divisible by `y`. */ - mod(y: UInt64 | number | string | bigint | UInt32) { + modV2(y: UInt64 | number | string | bigint | UInt32) { let y_ = UInt64.from(y); let rest = this.magnitude.divMod(y_).rest.value; - rest = Provable.if(this.isPositive(), rest, y_.value.sub(rest)); + rest = Provable.if(this.isNegative(), y_.value.sub(rest), rest); return new Int64(new UInt64(rest.value)); } @@ -1228,6 +1246,7 @@ class Int64 extends CircuitValue implements BalanceChange { } /** * @deprecated Use {@link isPositiveV2()} or {@link isNonNegativeV2()} instead. + * The current implementation actually tests for non-negativity. */ isPositive() { return this.sgn.isPositive(); @@ -1248,7 +1267,14 @@ class Int64 extends CircuitValue implements BalanceChange { return this.sgn.isPositive(); } - // TODO enable this check method in v2 + /** + * Checks if the value is negative (x < 0). + */ + isNegative() { + return this.sgn.isNegative(); + } + + // TODO enable this check method in v2, to force a unique representation of 0 static checkV2({ magnitude, sgn }: { magnitude: UInt64; sgn: Sign }) { // check that the magnitude is in range UInt64.check(magnitude); From c1ce7df268a42c6d987de85dde60e55ab5169d6b Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Sat, 25 May 2024 15:11:59 +0200 Subject: [PATCH 05/93] scaffold some types --- src/lib/provable/merkle-tree-indexed.ts | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/lib/provable/merkle-tree-indexed.ts diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts new file mode 100644 index 0000000000..9f16325459 --- /dev/null +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -0,0 +1,29 @@ +import { Field } from './field.js'; +import { Option } from './option.js'; +import { Struct } from './types/struct.js'; + +type IndexedMerkleMapBase = { + // (lower-level) method to insert a new leaf `(key, value)`. proves that `key` doesn't exist yet + insert(key: Field, value: Field): void; + + // (lower-level) method to update an existing leaf `(key, value)`. proves that the `key` exists. + update(key: Field, value: Field): void; + + // method that performs _either_ an insertion or update, depending on whether the key exists + set(key: Field, value: Field): void; + + // method to get a value from a key. returns an option to account for the key not existing + // note: this has to prove that the option's `isSome` is correct + get(key: Field): Option; // the optional `Field` here is the value + + // optional / nice-to-have: remove a key and its value from the tree; proves that the key is included. + // (implementation: leave a wasted leaf in place but skip it in the linked list encoding) + remove(key: Field): void; +}; + +class Leaf extends Struct({ + key: Field, + value: Field, + nextKey: Field, + nextIndex: Field, +}) {} From 061ef2249ff29aa70b5c03ddb733830185154e10 Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Sat, 25 May 2024 15:35:46 +0200 Subject: [PATCH 06/93] scaffold implementation --- src/lib/provable/merkle-tree-indexed.ts | 57 ++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 9f16325459..92eb499eca 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -1,8 +1,13 @@ -import { Field } from './field.js'; +import { Poseidon as PoseidonBigint } from '../../bindings/crypto/poseidon.js'; +import { Field } from './wrapped.js'; import { Option } from './option.js'; import { Struct } from './types/struct.js'; +import { InferValue } from 'src/bindings/lib/provable-generic.js'; +import { assert } from './gadgets/common.js'; type IndexedMerkleMapBase = { + root: Field; + // (lower-level) method to insert a new leaf `(key, value)`. proves that `key` doesn't exist yet insert(key: Field, value: Field): void; @@ -27,3 +32,53 @@ class Leaf extends Struct({ nextKey: Field, nextIndex: Field, }) {} + +class IndexedMerkleMap implements IndexedMerkleMapBase { + // data defining the provable interface of a tree + root: Field; + readonly height: number; + + // the raw data stored in the tree + readonly leaves: InferValue[] = []; + + // helpers structures + length: number; // length of the leaves array + readonly nodes: Field[][] = []; // for every level, an array of hashes + + /** + * Creates a new, empty Indexed Merkle Map, given its height. + */ + constructor(height: number) { + this.root = Field(zero(height - 1)); + } + + insert(key: Field, value: Field) { + assert(false, 'not implemented'); + } + + update(key: Field, value: Field) { + assert(false, 'not implemented'); + } + + set(key: Field, value: Field) { + assert(false, 'not implemented'); + } + + get(key: Field): Option { + assert(false, 'not implemented'); + } + + remove(key: Field) { + assert(false, 'not implemented'); + } +} + +// cache of zero hashes +const zeroes = [0n]; +function zero(level: number) { + for (let i = zeroes.length; i <= level; i++) { + let zero = zeroes[i - 1]; + zeroes[i] = PoseidonBigint.hash([zero, zero]); + } + return zeroes[level]; +} From c71a7badbba6e5aa8e70591c851b58f8be3d5e27 Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Sat, 25 May 2024 17:15:11 +0200 Subject: [PATCH 07/93] get / set internal nodes --- src/lib/provable/merkle-tree-indexed.ts | 59 ++++++++++++++++++++----- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 92eb499eca..a6bab1f4f3 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -41,15 +41,20 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // the raw data stored in the tree readonly leaves: InferValue[] = []; - // helpers structures - length: number; // length of the leaves array - readonly nodes: Field[][] = []; // for every level, an array of hashes + // helper structures + length: number = 0; // length of the leaves array + readonly nodes: (bigint | undefined)[][]; // for every level, an array of hashes /** * Creates a new, empty Indexed Merkle Map, given its height. */ constructor(height: number) { - this.root = Field(zero(height - 1)); + this.root = Field(empty(height - 1)); + + this.nodes = Array(height); + for (let level = 0; level < height; level++) { + this.nodes[level] = []; + } } insert(key: Field, value: Field) { @@ -71,14 +76,46 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { remove(key: Field) { assert(false, 'not implemented'); } + + // helper methods + + // invariant: for every node that is not undefined, its descendants are either empty or not undefined + private setLeafNode(index: number, leaf: bigint) { + this.nodes[0][index] = leaf; + + let isLeft = index % 2 === 0; + + for (let level = 1; level < this.height; level++) { + index = Math.floor(index / 2); + + let left = this.getNode(level - 1, index * 2, isLeft); + let right = this.getNode(level - 1, index * 2 + 1, !isLeft); + this.nodes[level][index] = PoseidonBigint.hash([left, right]); + + isLeft = index % 2 === 0; + } + } + + private getNode(level: number, index: number, nonEmpty: boolean) { + let node = this.nodes[level]?.[index]; + if (node === undefined) { + if (nonEmpty) + throw Error( + `node at level=${level}, index=${index} was expected to be known, but isn't.` + ); + node = empty(level); + } + return node; + } } -// cache of zero hashes -const zeroes = [0n]; -function zero(level: number) { - for (let i = zeroes.length; i <= level; i++) { - let zero = zeroes[i - 1]; - zeroes[i] = PoseidonBigint.hash([zero, zero]); +// cache of empty nodes (=: zero leaves and nodes with only empty nodes below them) +const emptyNodes = [0n]; + +function empty(level: number) { + for (let i = emptyNodes.length; i <= level; i++) { + let zero = emptyNodes[i - 1]; + emptyNodes[i] = PoseidonBigint.hash([zero, zero]); } - return zeroes[level]; + return emptyNodes[level]; } From 1859512930d9bac9e691ae258570fe3a5852d3cd Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Sat, 25 May 2024 17:23:02 +0200 Subject: [PATCH 08/93] restructure --- src/lib/provable/merkle-tree-indexed.ts | 30 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index a6bab1f4f3..18512d9c09 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -4,6 +4,7 @@ import { Option } from './option.js'; import { Struct } from './types/struct.js'; import { InferValue } from 'src/bindings/lib/provable-generic.js'; import { assert } from './gadgets/common.js'; +import { Unconstrained } from './types/unconstrained.js'; type IndexedMerkleMapBase = { root: Field; @@ -36,14 +37,16 @@ class Leaf extends Struct({ class IndexedMerkleMap implements IndexedMerkleMapBase { // data defining the provable interface of a tree root: Field; + length: Field; // length of the leaves array readonly height: number; - // the raw data stored in the tree - readonly leaves: InferValue[] = []; + // the raw data stored in the tree, plus helper structures + readonly data: Unconstrained<{ + readonly leaves: InferValue[]; - // helper structures - length: number = 0; // length of the leaves array - readonly nodes: (bigint | undefined)[][]; // for every level, an array of hashes + // for every level, an array of hashes + readonly nodes: (bigint | undefined)[][]; + }>; /** * Creates a new, empty Indexed Merkle Map, given its height. @@ -51,10 +54,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { constructor(height: number) { this.root = Field(empty(height - 1)); - this.nodes = Array(height); + let nodes: (bigint | undefined)[][] = Array(height); for (let level = 0; level < height; level++) { - this.nodes[level] = []; + nodes[level] = []; } + + this.length = Field(0); + let leaves: InferValue[] = []; + + this.data = Unconstrained.from({ leaves, nodes }); } insert(key: Field, value: Field) { @@ -81,8 +89,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // invariant: for every node that is not undefined, its descendants are either empty or not undefined private setLeafNode(index: number, leaf: bigint) { - this.nodes[0][index] = leaf; + let nodes = this.data.get().nodes; + nodes[0][index] = leaf; let isLeft = index % 2 === 0; for (let level = 1; level < this.height; level++) { @@ -90,14 +99,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { let left = this.getNode(level - 1, index * 2, isLeft); let right = this.getNode(level - 1, index * 2 + 1, !isLeft); - this.nodes[level][index] = PoseidonBigint.hash([left, right]); + nodes[level][index] = PoseidonBigint.hash([left, right]); isLeft = index % 2 === 0; } } private getNode(level: number, index: number, nonEmpty: boolean) { - let node = this.nodes[level]?.[index]; + let nodes = this.data.get().nodes; + let node = nodes[level]?.[index]; if (node === undefined) { if (nonEmpty) throw Error( From d2e8d9185d47794994a4cf3a8790bc944853b9a5 Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Mon, 27 May 2024 20:56:36 +0200 Subject: [PATCH 09/93] introduce low nodes to the data structure --- src/lib/provable/merkle-tree-indexed.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 18512d9c09..b126e920c5 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -33,6 +33,7 @@ class Leaf extends Struct({ nextKey: Field, nextIndex: Field, }) {} +type LeafValue = InferValue; class IndexedMerkleMap implements IndexedMerkleMapBase { // data defining the provable interface of a tree @@ -42,10 +43,13 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // the raw data stored in the tree, plus helper structures readonly data: Unconstrained<{ - readonly leaves: InferValue[]; + readonly leaves: LeafValue[]; // for every level, an array of hashes readonly nodes: (bigint | undefined)[][]; + + // sorted list of low nodes + readonly lowNodes: LeafValue[]; }>; /** @@ -60,9 +64,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } this.length = Field(0); - let leaves: InferValue[] = []; - - this.data = Unconstrained.from({ leaves, nodes }); + let leaves: LeafValue[] = []; + let lowNodes: LeafValue[] = []; + this.data = Unconstrained.from({ leaves, lowNodes, nodes }); } insert(key: Field, value: Field) { @@ -87,6 +91,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods + private getLowNode(key: bigint) {} + // invariant: for every node that is not undefined, its descendants are either empty or not undefined private setLeafNode(index: number, leaf: bigint) { let nodes = this.data.get().nodes; From 320ebd235ac0bb2dbafb65b32c05ff53a5edd3d0 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 08:52:57 +0200 Subject: [PATCH 10/93] fixup --- src/lib/provable/int.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index 85628d8b63..e177663469 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -1259,19 +1259,21 @@ class Int64 extends CircuitValue implements BalanceChange { return this.magnitude.equals(UInt64.zero).not().and(this.sgn.isPositive()); } + // TODO add this when `checkV2` is enabled + // right now it would be misleading /** * Checks if the value is non-negative (x >= 0). */ - isNonNegativeV2() { - // this will be the correct logic when `checkV2` is enabled - return this.sgn.isPositive(); - } + // isNonNegativeV2() { + // // this will be the correct logic when `checkV2` is enabled + // return this.sgn.isPositive(); + // } /** * Checks if the value is negative (x < 0). */ isNegative() { - return this.sgn.isNegative(); + return this.magnitude.equals(UInt64.zero).not().and(this.sgn.isPositive()); } // TODO enable this check method in v2, to force a unique representation of 0 From 264080900a16a19e31509ef479e7e2fa9c56fce6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 08:53:47 +0200 Subject: [PATCH 11/93] fixup --- src/lib/provable/int.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index e177663469..26ec3a30f1 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -1273,7 +1273,7 @@ class Int64 extends CircuitValue implements BalanceChange { * Checks if the value is negative (x < 0). */ isNegative() { - return this.magnitude.equals(UInt64.zero).not().and(this.sgn.isPositive()); + return this.magnitude.equals(UInt64.zero).not().and(this.sgn.isNegative()); } // TODO enable this check method in v2, to force a unique representation of 0 From 338ee5d59defa90933966536c7d0114de80afd33 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 08:57:11 +0200 Subject: [PATCH 12/93] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3ff0d046..df9e969623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased](https://github.com/o1-labs/o1js/compare/54d6545bf...HEAD) +### Deprecated + +- `Int64.isPositive()` and `Int64.mod()` deprecated because they behave incorrectly on `-0` https://github.com/o1-labs/o1js/pull/1660 + - This can pose an attack surface, since it is easy to maliciously pick either the `+0` or the `-0` representation + - Use `Int64.isPositiveV2()` and `Int64.modV2()` instead + ## [1.3.0](https://github.com/o1-labs/o1js/compare/6a1012162...54d6545bf) ### Added From 9bb7f7395b51c834b042e24b03c0b42ad2984b1c Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 09:06:51 +0200 Subject: [PATCH 13/93] be more correct without introducing more methods that we break in v2 --- src/lib/provable/int.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index 26ec3a30f1..a471992db4 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -1223,7 +1223,10 @@ class Int64 extends CircuitValue implements BalanceChange { modV2(y: UInt64 | number | string | bigint | UInt32) { let y_ = UInt64.from(y); let rest = this.magnitude.divMod(y_).rest.value; - rest = Provable.if(this.isNegative(), y_.value.sub(rest), rest); + let isNonNegative = this.magnitude + .equals(UInt64.zero) + .or(this.sgn.isPositive()); + rest = Provable.if(isNonNegative, rest, y_.value.sub(rest)); return new Int64(new UInt64(rest.value)); } @@ -1245,8 +1248,8 @@ class Int64 extends CircuitValue implements BalanceChange { this.toField().assertEquals(y_.toField(), message); } /** - * @deprecated Use {@link isPositiveV2()} or {@link isNonNegativeV2()} instead. - * The current implementation actually tests for non-negativity. + * @deprecated Use {@link isPositiveV2} instead. + * The current implementation actually tests for non-negativity, but is wrong for the negative representation of 0. */ isPositive() { return this.sgn.isPositive(); @@ -1260,21 +1263,22 @@ class Int64 extends CircuitValue implements BalanceChange { } // TODO add this when `checkV2` is enabled - // right now it would be misleading + // then it will be the correct logic; right now it would be misleading /** * Checks if the value is non-negative (x >= 0). */ // isNonNegativeV2() { - // // this will be the correct logic when `checkV2` is enabled // return this.sgn.isPositive(); // } + // TODO add this when `checkV2` is enabled + // then it will be the correct logic; right now it would be misleading /** * Checks if the value is negative (x < 0). */ - isNegative() { - return this.magnitude.equals(UInt64.zero).not().and(this.sgn.isNegative()); - } + // isNegative() { + // return this.sgn.isNegative(); + // } // TODO enable this check method in v2, to force a unique representation of 0 static checkV2({ magnitude, sgn }: { magnitude: UInt64; sgn: Sign }) { From 1f9b7077634db00c08d925d0e6747d329f45a6f9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 10:13:23 +0200 Subject: [PATCH 14/93] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df9e969623..ac3634036d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - `Int64.isPositive()` and `Int64.mod()` deprecated because they behave incorrectly on `-0` https://github.com/o1-labs/o1js/pull/1660 - This can pose an attack surface, since it is easy to maliciously pick either the `+0` or the `-0` representation - Use `Int64.isPositiveV2()` and `Int64.modV2()` instead + - Also deprecated `Int64.neg()` in favor of `Int64.negV2()`, for compatibility with v2 version of `Int64` that will use `Int64.checkV2()` ## [1.3.0](https://github.com/o1-labs/o1js/compare/6a1012162...54d6545bf) From ecc495cdb4459d34cbc64ec8811e2154abd53e2f Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 10:48:23 +0200 Subject: [PATCH 15/93] bisection algorithm --- src/lib/provable/merkle-tree-indexed.ts | 44 +++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index b126e920c5..ae375b7005 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -135,3 +135,47 @@ function empty(level: number) { } return emptyNodes[level]; } + +// helper + +/** + * Bisect indices in an array of unique values that is sorted in ascending order. + * + * `getValue()` returns the value at the given index. + * + * We return + * `lowIndex` := max { i in [0, length) | getValue(i) <= target } + * `foundValue` := whether `getValue(lowIndex) == target` + */ +function bisectUnique( + target: bigint, + getValue: (index: number) => bigint, + length: number +): { + lowIndex: number; + foundValue: boolean; +} { + // invariants: iLow <= (target index) <= iHigh and 0 <= iLow < length + let [iLow, iHigh] = [0, length]; + while (true) { + if (iHigh === iLow) { + let foundValue = getValue(iLow) === target; + return { lowIndex: iLow, foundValue }; + } else if (iHigh - iLow === 1) { + // this gets us to the iHigh = iLow case in the next iteration + if (getValue(iHigh) <= target) iLow = iHigh; + else iHigh = iLow; + } else { + // because of the other cases, we know that iLow < iMid < iHigh + let iMid = Math.floor((iLow + iHigh) / 2); + if (getValue(iMid) <= target) { + iLow = iMid; + } else { + // we can use iMid - 1 because + // - iMid > iLow >= 0, and + // - iMid is no longer a candidate index that we can return + iHigh = iMid - 1; + } + } + } +} From df3c66a6777af9b442ac58a04664e340d6993435 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 11:16:43 +0200 Subject: [PATCH 16/93] fix and simplify algorithm --- src/lib/provable/merkle-tree-indexed.ts | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index ae375b7005..82d706c665 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -155,27 +155,27 @@ function bisectUnique( lowIndex: number; foundValue: boolean; } { - // invariants: iLow <= (target index) <= iHigh and 0 <= iLow < length - let [iLow, iHigh] = [0, length]; + let [iLow, iHigh] = [0, length - 1]; + if (getValue(iLow) > target) return { lowIndex: -1, foundValue: false }; + if (getValue(iHigh) < target) return { lowIndex: iHigh, foundValue: false }; + + // invariant: 0 <= iLow <= lowIndex <= iHigh < length + // since we are either returning or reducing (iHigh - iLow), we'll eventually terminate correctly while (true) { if (iHigh === iLow) { - let foundValue = getValue(iLow) === target; - return { lowIndex: iLow, foundValue }; - } else if (iHigh - iLow === 1) { - // this gets us to the iHigh = iLow case in the next iteration - if (getValue(iHigh) <= target) iLow = iHigh; - else iHigh = iLow; + return { lowIndex: iLow, foundValue: getValue(iLow) === target }; + } + // either iLow + 1 = iHigh = iMid, or iLow < iMid < iHigh + // in both cases, the range gets strictly smaller + let iMid = Math.ceil((iLow + iHigh) / 2); + if (getValue(iMid) <= target) { + // iMid is in the candidate set, and strictly larger than iLow + // preserves iLow <= lowIndex + iLow = iMid; } else { - // because of the other cases, we know that iLow < iMid < iHigh - let iMid = Math.floor((iLow + iHigh) / 2); - if (getValue(iMid) <= target) { - iLow = iMid; - } else { - // we can use iMid - 1 because - // - iMid > iLow >= 0, and - // - iMid is no longer a candidate index that we can return - iHigh = iMid - 1; - } + // iMid is no longer in the candidate set, so we can exclude it right away + // preserves lowIndex <= iHigh + iHigh = iMid - 1; } } } From 5ccd155a3ad6f3e476dcc4b82b583cd21df12e5d Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 11:57:09 +0200 Subject: [PATCH 17/93] find node --- src/lib/provable/merkle-tree-indexed.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 82d706c665..0d7439449b 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -49,6 +49,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { readonly nodes: (bigint | undefined)[][]; // sorted list of low nodes + // we always have lowNodes[n-1].nextKey = 0 = lowNodes[0].key + // for i=0,...n-2, lowNodes[i].nextKey = lowNodes[i+1].key readonly lowNodes: LeafValue[]; }>; @@ -91,7 +93,19 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods - private getLowNode(key: bigint) {} + private findNode(key: bigint) { + assert(key > 0n, 'key must be positive'); + + let lowNodes = this.data.get().lowNodes; + let { lowIndex, foundValue } = bisectUnique( + key, + (i) => lowNodes[i].key, + lowNodes.length + ); + let lowNode = foundValue ? lowNodes[lowIndex - 1] : lowNodes[lowIndex]; + let thisNode = foundValue ? lowNodes[lowIndex] : undefined; + return { lowNode, thisNode }; + } // invariant: for every node that is not undefined, its descendants are either empty or not undefined private setLeafNode(index: number, leaf: bigint) { From 977d0d5acf94ec4353e34a8036eb842bc85476e2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 15:11:49 +0200 Subject: [PATCH 18/93] inclusion proof --- src/lib/provable/merkle-tree-indexed.ts | 73 +++++++++++++++++++------ src/lib/provable/merkle-tree.ts | 1 + 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 0d7439449b..2ab19e5ce6 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -1,10 +1,13 @@ import { Poseidon as PoseidonBigint } from '../../bindings/crypto/poseidon.js'; -import { Field } from './wrapped.js'; +import { Bool, Field } from './wrapped.js'; import { Option } from './option.js'; import { Struct } from './types/struct.js'; import { InferValue } from 'src/bindings/lib/provable-generic.js'; import { assert } from './gadgets/common.js'; import { Unconstrained } from './types/unconstrained.js'; +import { Provable } from './provable.js'; +import { Poseidon } from './crypto/poseidon.js'; +import { maybeSwap } from './merkle-tree.js'; type IndexedMerkleMapBase = { root: Field; @@ -32,6 +35,8 @@ class Leaf extends Struct({ value: Field, nextKey: Field, nextIndex: Field, + + index: Unconstrained.provableWithEmpty(0), }) {} type LeafValue = InferValue; @@ -43,15 +48,16 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // the raw data stored in the tree, plus helper structures readonly data: Unconstrained<{ + // leaves sorted by index readonly leaves: LeafValue[]; + // leaves sorted by key, with a circular linked list encoded by nextKey + // we always have sortedLeaves[n-1].nextKey = 0 = sortedLeaves[0].key + // for i=0,...n-2, sortedLeaves[i].nextKey = sortedLeaves[i+1].key + readonly sortedLeaves: LeafValue[]; + // for every level, an array of hashes readonly nodes: (bigint | undefined)[][]; - - // sorted list of low nodes - // we always have lowNodes[n-1].nextKey = 0 = lowNodes[0].key - // for i=0,...n-2, lowNodes[i].nextKey = lowNodes[i+1].key - readonly lowNodes: LeafValue[]; }>; /** @@ -67,12 +73,19 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.length = Field(0); let leaves: LeafValue[] = []; - let lowNodes: LeafValue[] = []; - this.data = Unconstrained.from({ leaves, lowNodes, nodes }); + let sortedLeaves: LeafValue[] = []; + this.data = Unconstrained.from({ leaves, sortedLeaves, nodes }); } insert(key: Field, value: Field) { - assert(false, 'not implemented'); + let lowNode = Provable.witness(Leaf, () => this.findNode(key).lowNode); + + // prove that the key doesn't exist yet and we have a valid low node + lowNode.key.assertLessThan(key); + key.assertLessThan(lowNode.nextKey); + // TODO + + let newIndex = this.length; } update(key: Field, value: Field) { @@ -93,17 +106,44 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods - private findNode(key: bigint) { - assert(key > 0n, 'key must be positive'); + proveInclusion(leaf: Leaf, message?: string) { + let node = Poseidon.hashPacked(Leaf, leaf); + let index = Unconstrained.witness(() => leaf.index.get()); + + for (let level = 1; level < this.height; level++) { + // in every iteration, we witness a sibling and hash it to get the parent node + let isLeft = Provable.witness(Bool, () => index.get() % 2 === 0); + let sibling = Provable.witness(Field, () => { + let i = index.get(); + let isLeft = i % 2 === 0; + return this.getNode(level - 1, isLeft ? i + 1 : i - 1, false); + }); + let [left, right] = maybeSwap(isLeft, node, sibling); + node = Poseidon.hash([left, right]); + index.updateAsProver((i) => i >> 1); + } + + // now, `node` is the root of the tree, and we can check it against the actual root + node.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); + } + + private findNode(key_: Field | bigint) { + let key = typeof key_ === 'bigint' ? key_ : key_.toBigInt(); + assert(key >= 0n, 'key must be positive'); + let leaves = this.data.get().sortedLeaves; + + // this case is typically invalid, but we want to handle it gracefully here + // and reject it using comparison constraints + if (key === 0n) + return { lowNode: leaves[leaves.length - 1], thisNode: leaves[0] }; - let lowNodes = this.data.get().lowNodes; let { lowIndex, foundValue } = bisectUnique( key, - (i) => lowNodes[i].key, - lowNodes.length + (i) => leaves[i].key, + leaves.length ); - let lowNode = foundValue ? lowNodes[lowIndex - 1] : lowNodes[lowIndex]; - let thisNode = foundValue ? lowNodes[lowIndex] : undefined; + let lowNode = foundValue ? leaves[lowIndex - 1] : leaves[lowIndex]; + let thisNode = foundValue ? leaves[lowIndex] : undefined; return { lowNode, thisNode }; } @@ -170,6 +210,7 @@ function bisectUnique( foundValue: boolean; } { let [iLow, iHigh] = [0, length - 1]; + // handle out of bounds if (getValue(iLow) > target) return { lowIndex: -1, foundValue: false }; if (getValue(iHigh) < target) return { lowIndex: iHigh, foundValue: false }; diff --git a/src/lib/provable/merkle-tree.ts b/src/lib/provable/merkle-tree.ts index 090d244471..a1be3b0a91 100644 --- a/src/lib/provable/merkle-tree.ts +++ b/src/lib/provable/merkle-tree.ts @@ -246,6 +246,7 @@ function MerkleWitness(height: number): typeof BaseMerkleWitness { return MerkleWitness_; } +// swap two values if the boolean is false, otherwise keep them as they are // more efficient than 2x `Provable.if()` by reusing an intermediate variable function maybeSwap(b: Bool, x: Field, y: Field): [Field, Field] { let m = b.toField().mul(x.sub(y)); // b*(x - y) From 5395145ae34a218e9d08faf2105e31152413f598 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 16:55:33 +0200 Subject: [PATCH 19/93] insert leaf --- src/lib/provable/merkle-tree-indexed.ts | 106 +++++++++++++++++++----- 1 file changed, 86 insertions(+), 20 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 2ab19e5ce6..968c7d95bf 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -37,7 +37,22 @@ class Leaf extends Struct({ nextIndex: Field, index: Unconstrained.provableWithEmpty(0), -}) {} + sortedIndex: Unconstrained.provableWithEmpty(0), +}) { + static hash({ + key, + value, + nextKey, + nextIndex, + }: { + key: Field; + value: Field; + nextKey: Field; + nextIndex: Field; + }) { + return Poseidon.hash([key, value, nextKey, nextIndex]); + } +} type LeafValue = InferValue; class IndexedMerkleMap implements IndexedMerkleMapBase { @@ -78,14 +93,21 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } insert(key: Field, value: Field) { - let lowNode = Provable.witness(Leaf, () => this.findNode(key).lowNode); + // prove that the key doesn't exist yet by presenting a valid low node + let lowNode = Provable.witness(Leaf, () => this.findLeaf(key).low); + this.proveInclusion(lowNode, 'Invalid low node'); + lowNode.key.assertLessThan(key, 'Invalid low node'); - // prove that the key doesn't exist yet and we have a valid low node - lowNode.key.assertLessThan(key); - key.assertLessThan(lowNode.nextKey); - // TODO + // if the key does exist, we have lowNode.nextKey == key, and this line fails + key.assertLessThan(lowNode.nextKey, 'Key already exists in the tree'); - let newIndex = this.length; + // update low node + let nextIndex = this.length; + let newLowNode = { ...lowNode, nextKey: key, nextIndex }; + this.updateLeaf(newLowNode); + + // append new leaf + this.appendLeaf(lowNode, { key, value }); } update(key: Field, value: Field) { @@ -107,44 +129,88 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods proveInclusion(leaf: Leaf, message?: string) { - let node = Poseidon.hashPacked(Leaf, leaf); - let index = Unconstrained.witness(() => leaf.index.get()); + let node = Leaf.hash(leaf); + let index = leaf.index; + let root = this.computeRoot(node, index); + root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); + } - for (let level = 1; level < this.height; level++) { + updateLeaf(leaf: Leaf) { + let node = Leaf.hash(leaf); + let index = leaf.index; + this.root = this.computeRoot(node, index); + Provable.asProver(() => { + // update internal hash nodes + this.setLeafNode(index.get(), node.toBigInt()); + }); + } + + appendLeaf(low: Leaf, { key, value }: { key: Field; value: Field }) { + let index = Unconstrained.witness(() => Number(this.length.toBigInt())); + + // update root and length + let leaf = { key, value, nextKey: low.nextKey, nextIndex: low.nextIndex }; + let node = Leaf.hash(leaf); + this.root = this.computeRoot(node, index); + this.length = this.length.add(1); + + Provable.asProver(() => { + // update internal hash nodes + this.setLeafNode(index.get(), node.toBigInt()); + + // update leaf lists + let sortedIndex = low.sortedIndex.get() + 1; + let leafValue = Leaf.toValue({ + ...leaf, + index, + sortedIndex: Unconstrained.from(sortedIndex), + }); + + let { leaves, sortedLeaves } = this.data.get(); + leaves.push(leafValue); + sortedLeaves.splice(sortedIndex, 0, leafValue); + }); + } + + computeRoot(node: Field, index: Unconstrained) { + for (let level = 0; level < this.height - 1; level++) { // in every iteration, we witness a sibling and hash it to get the parent node let isLeft = Provable.witness(Bool, () => index.get() % 2 === 0); let sibling = Provable.witness(Field, () => { let i = index.get(); let isLeft = i % 2 === 0; - return this.getNode(level - 1, isLeft ? i + 1 : i - 1, false); + return this.getNode(level, isLeft ? i + 1 : i - 1, false); }); let [left, right] = maybeSwap(isLeft, node, sibling); node = Poseidon.hash([left, right]); index.updateAsProver((i) => i >> 1); } - - // now, `node` is the root of the tree, and we can check it against the actual root - node.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); + // now, `node` is the root of the tree + return node; } - private findNode(key_: Field | bigint) { + /** + * Given a key, returns both the low node and the node that contains the key. + * + * Assumes to run outside provable code. + */ + private findLeaf(key_: Field | bigint) { let key = typeof key_ === 'bigint' ? key_ : key_.toBigInt(); assert(key >= 0n, 'key must be positive'); let leaves = this.data.get().sortedLeaves; // this case is typically invalid, but we want to handle it gracefully here // and reject it using comparison constraints - if (key === 0n) - return { lowNode: leaves[leaves.length - 1], thisNode: leaves[0] }; + if (key === 0n) return { low: leaves[leaves.length - 1], self: leaves[0] }; let { lowIndex, foundValue } = bisectUnique( key, (i) => leaves[i].key, leaves.length ); - let lowNode = foundValue ? leaves[lowIndex - 1] : leaves[lowIndex]; - let thisNode = foundValue ? leaves[lowIndex] : undefined; - return { lowNode, thisNode }; + let low = foundValue ? leaves[lowIndex - 1] : leaves[lowIndex]; + let self = foundValue ? leaves[lowIndex] : undefined; + return { low, self }; } // invariant: for every node that is not undefined, its descendants are either empty or not undefined From cc3093915523f61f7cd6a3bb851b8f3fdafe3b3d Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 17:02:14 +0200 Subject: [PATCH 20/93] fix update leaf, some cleanup --- src/lib/provable/merkle-tree-indexed.ts | 26 ++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 968c7d95bf..50c87369bc 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -102,9 +102,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { key.assertLessThan(lowNode.nextKey, 'Key already exists in the tree'); // update low node - let nextIndex = this.length; - let newLowNode = { ...lowNode, nextKey: key, nextIndex }; - this.updateLeaf(newLowNode); + this.updateLeaf(lowNode, { nextKey: key, nextIndex: this.length }); // append new leaf this.appendLeaf(lowNode, { key, value }); @@ -135,16 +133,34 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); } - updateLeaf(leaf: Leaf) { - let node = Leaf.hash(leaf); + /** + * Update existing leaf with new pointers + */ + updateLeaf( + leaf: Leaf, + { nextKey, nextIndex }: { nextKey: Field; nextIndex: Field } + ) { + // update root + let newLeaf = { ...leaf, nextKey, nextIndex }; + let node = Leaf.hash(newLeaf); let index = leaf.index; this.root = this.computeRoot(node, index); + Provable.asProver(() => { // update internal hash nodes this.setLeafNode(index.get(), node.toBigInt()); + + // update leaf lists + let { leaves, sortedLeaves } = this.data.get(); + let leafValue = Leaf.toValue(newLeaf); + leaves[index.get()] = leafValue; + sortedLeaves[leaf.sortedIndex.get()] = leafValue; }); } + /** + * Append a new leaf based on the pointers of the previous low node + */ appendLeaf(low: Leaf, { key, value }: { key: Field; value: Field }) { let index = Unconstrained.witness(() => Number(this.length.toBigInt())); From af33aec7c211ba8274459cd6d8a32d4ccce707b5 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 17:20:58 +0200 Subject: [PATCH 21/93] update --- src/lib/provable/merkle-tree-indexed.ts | 39 +++++++++++++++---------- src/lib/util/errors.ts | 12 ++++++++ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 50c87369bc..d827b74479 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -8,6 +8,7 @@ import { Unconstrained } from './types/unconstrained.js'; import { Provable } from './provable.js'; import { Poseidon } from './crypto/poseidon.js'; import { maybeSwap } from './merkle-tree.js'; +import { assertDefined } from '../util/errors.js'; type IndexedMerkleMapBase = { root: Field; @@ -30,6 +31,13 @@ type IndexedMerkleMapBase = { remove(key: Field): void; }; +type BaseLeaf = { + key: Field; + value: Field; + nextKey: Field; + nextIndex: Field; +}; + class Leaf extends Struct({ key: Field, value: Field, @@ -39,17 +47,7 @@ class Leaf extends Struct({ index: Unconstrained.provableWithEmpty(0), sortedIndex: Unconstrained.provableWithEmpty(0), }) { - static hash({ - key, - value, - nextKey, - nextIndex, - }: { - key: Field; - value: Field; - nextKey: Field; - nextIndex: Field; - }) { + static hash({ key, value, nextKey, nextIndex }: BaseLeaf) { return Poseidon.hash([key, value, nextKey, nextIndex]); } } @@ -109,7 +107,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } update(key: Field, value: Field) { - assert(false, 'not implemented'); + // prove that the key exists by presenting a leaf that contains it + let self = Provable.witness(Leaf, () => + assertDefined(this.findLeaf(key).self, 'Key does not exist in the tree') + ); + this.proveInclusion(self, 'Key does not exist in the tree'); + self.key.assertEquals(key, 'Invalid leaf'); + + // update leaf + this.updateLeaf(self, { value }); } set(key: Field, value: Field) { @@ -134,14 +140,16 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } /** - * Update existing leaf with new pointers + * Update existing leaf. + * + * Note: we never update the key of a leaf. */ updateLeaf( leaf: Leaf, - { nextKey, nextIndex }: { nextKey: Field; nextIndex: Field } + partialLeaf: Partial<{ value: Field; nextKey: Field; nextIndex: Field }> ) { // update root - let newLeaf = { ...leaf, nextKey, nextIndex }; + let newLeaf = { ...leaf, ...partialLeaf }; let node = Leaf.hash(newLeaf); let index = leaf.index; this.root = this.computeRoot(node, index); @@ -188,6 +196,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { }); } + // TODO this does not provably use the index! computeRoot(node: Field, index: Unconstrained) { for (let level = 0; level < this.height - 1; level++) { // in every iteration, we witness a sibling and hash it to get the parent node diff --git a/src/lib/util/errors.ts b/src/lib/util/errors.ts index 3fe584eac2..b832c7be6b 100644 --- a/src/lib/util/errors.ts +++ b/src/lib/util/errors.ts @@ -4,6 +4,7 @@ export { prettifyStacktrace, prettifyStacktracePromise, assert, + assertDefined, }; /** @@ -280,3 +281,14 @@ function assert( ): asserts condition { if (!condition) throw Bug(message); } + +/** + * Assert that the value is not undefined, return the value. + */ +function assertDefined( + value: T | undefined, + message = 'Input value is undefined.' +): T { + if (value !== undefined) throw Bug(message); + return value as T; +} From 171d168cb9dab128f78ae2840084e132b1c0ff25 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 17:47:00 +0200 Subject: [PATCH 22/93] represent index in constraints --- src/lib/provable/merkle-tree-indexed.ts | 59 +++++++++++++++---------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index d827b74479..ce3c6c5a59 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -39,15 +39,17 @@ type BaseLeaf = { }; class Leaf extends Struct({ - key: Field, value: Field, + + key: Field, nextKey: Field, + + index: Field, nextIndex: Field, - index: Unconstrained.provableWithEmpty(0), sortedIndex: Unconstrained.provableWithEmpty(0), }) { - static hash({ key, value, nextKey, nextIndex }: BaseLeaf) { + static hashNode({ key, value, nextKey, nextIndex }: BaseLeaf) { return Poseidon.hash([key, value, nextKey, nextIndex]); } } @@ -77,6 +79,11 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { * Creates a new, empty Indexed Merkle Map, given its height. */ constructor(height: number) { + assert(height > 0, 'height must be positive'); + assert( + height < 53, + 'height must be less than 53, so that we can use 64-bit floats to represent indices.' + ); this.root = Field(empty(height - 1)); let nodes: (bigint | undefined)[][] = Array(height); @@ -133,16 +140,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods proveInclusion(leaf: Leaf, message?: string) { - let node = Leaf.hash(leaf); - let index = leaf.index; - let root = this.computeRoot(node, index); + let node = Leaf.hashNode(leaf); + let root = this.computeRoot(leaf.index, node); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); } /** * Update existing leaf. * - * Note: we never update the key of a leaf. + * Note: we never update the key or index of a leaf. */ updateLeaf( leaf: Leaf, @@ -150,18 +156,18 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { ) { // update root let newLeaf = { ...leaf, ...partialLeaf }; - let node = Leaf.hash(newLeaf); - let index = leaf.index; - this.root = this.computeRoot(node, index); + let node = Leaf.hashNode(newLeaf); + this.root = this.computeRoot(leaf.index, node); Provable.asProver(() => { // update internal hash nodes - this.setLeafNode(index.get(), node.toBigInt()); + let i = Number(leaf.index.toBigInt()); + this.setLeafNode(i, node.toBigInt()); // update leaf lists let { leaves, sortedLeaves } = this.data.get(); let leafValue = Leaf.toValue(newLeaf); - leaves[index.get()] = leafValue; + leaves[i] = leafValue; sortedLeaves[leaf.sortedIndex.get()] = leafValue; }); } @@ -170,17 +176,17 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { * Append a new leaf based on the pointers of the previous low node */ appendLeaf(low: Leaf, { key, value }: { key: Field; value: Field }) { - let index = Unconstrained.witness(() => Number(this.length.toBigInt())); - // update root and length + let index = this.length; let leaf = { key, value, nextKey: low.nextKey, nextIndex: low.nextIndex }; - let node = Leaf.hash(leaf); - this.root = this.computeRoot(node, index); + let node = Leaf.hashNode(leaf); + this.root = this.computeRoot(index, node); this.length = this.length.add(1); Provable.asProver(() => { // update internal hash nodes - this.setLeafNode(index.get(), node.toBigInt()); + let i = Number(index.toBigInt()); + this.setLeafNode(i, node.toBigInt()); // update leaf lists let sortedIndex = low.sortedIndex.get() + 1; @@ -196,19 +202,24 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { }); } - // TODO this does not provably use the index! - computeRoot(node: Field, index: Unconstrained) { + /** + * Compute the root given a leaf node and its index. + */ + computeRoot(index: Field, node: Field) { + let indexU = Unconstrained.from(Number(index.toBigInt())); + let indexBits = index.toBits(this.height - 1); + for (let level = 0; level < this.height - 1; level++) { // in every iteration, we witness a sibling and hash it to get the parent node - let isLeft = Provable.witness(Bool, () => index.get() % 2 === 0); + let isRight = indexBits[level]; let sibling = Provable.witness(Field, () => { - let i = index.get(); - let isLeft = i % 2 === 0; + let i = indexU.get(); + let isLeft = !isRight.toBoolean(); return this.getNode(level, isLeft ? i + 1 : i - 1, false); }); - let [left, right] = maybeSwap(isLeft, node, sibling); + let [right, left] = maybeSwap(isRight, node, sibling); node = Poseidon.hash([left, right]); - index.updateAsProver((i) => i >> 1); + indexU.updateAsProver((i) => i >> 1); } // now, `node` is the root of the tree return node; From 472ba2441bf87e51bd30fe3df9489081cd4736bc Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 18:20:17 +0200 Subject: [PATCH 23/93] start on set --- src/lib/provable/merkle-tree-indexed.ts | 79 ++++++++++++++++++++----- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index ce3c6c5a59..b3b3e4383a 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -99,18 +99,18 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { insert(key: Field, value: Field) { // prove that the key doesn't exist yet by presenting a valid low node - let lowNode = Provable.witness(Leaf, () => this.findLeaf(key).low); - this.proveInclusion(lowNode, 'Invalid low node'); - lowNode.key.assertLessThan(key, 'Invalid low node'); + let low = Provable.witness(Leaf, () => this.findLeaf(key).low); + this.proveInclusion(low, 'Invalid low node'); + low.key.assertLessThan(key, 'Invalid low node'); // if the key does exist, we have lowNode.nextKey == key, and this line fails - key.assertLessThan(lowNode.nextKey, 'Key already exists in the tree'); + key.assertLessThan(low.nextKey, 'Key already exists in the tree'); // update low node - this.updateLeaf(lowNode, { nextKey: key, nextIndex: this.length }); + this.updateLeaf(low, { nextKey: key, nextIndex: this.length }); // append new leaf - this.appendLeaf(lowNode, { key, value }); + this.appendLeaf(low, { key, value }); } update(key: Field, value: Field) { @@ -126,7 +126,28 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } set(key: Field, value: Field) { - assert(false, 'not implemented'); + // prove whether the key exists or not, by showing a valid low node and checking if it points to the key + let low = Provable.witness(Leaf, () => this.findLeaf(key).low); + this.proveInclusion(low, 'Invalid low node'); + low.key.assertLessThan(key, 'Invalid low node'); + key.assertLessThanOrEqual(low.nextKey, 'Invalid low node'); + + // the key exists iff lowNode.nextKey == key + let keyExists = low.nextKey.equals(key); + + // the leaf's index depends on whether it exists + let index = Provable.if(keyExists, low.nextIndex, this.length); + + // update low node, or leave it as is + let newLow = { ...low, nextKey: key, nextIndex: index }; + let nodeLow = Leaf.hashNode(newLow); + this.root = this.computeRoot(low.index, nodeLow); + + // update leaf, or append a new one + // TODO damn I think we need another inclusion proof for the previous leaf + throw Error('not implemented'); + let TODO: any = {}; + this.setLeafUnconstrained(keyExists, low, TODO); } get(key: Field): Option { @@ -140,6 +161,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods proveInclusion(leaf: Leaf, message?: string) { + // TODO: here, we don't actually care about the index, so we could add a mode where `computeRoot()` doesn't prove it let node = Leaf.hashNode(leaf); let root = this.computeRoot(leaf.index, node); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); @@ -150,10 +172,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { * * Note: we never update the key or index of a leaf. */ - updateLeaf( - leaf: Leaf, - partialLeaf: Partial<{ value: Field; nextKey: Field; nextIndex: Field }> - ) { + updateLeaf(leaf: Leaf, partialLeaf: Partial) { // update root let newLeaf = { ...leaf, ...partialLeaf }; let node = Leaf.hashNode(newLeaf); @@ -189,16 +208,48 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafNode(i, node.toBigInt()); // update leaf lists - let sortedIndex = low.sortedIndex.get() + 1; + let iSorted = low.sortedIndex.get() + 1; let leafValue = Leaf.toValue({ ...leaf, index, - sortedIndex: Unconstrained.from(sortedIndex), + sortedIndex: Unconstrained.from(iSorted), }); let { leaves, sortedLeaves } = this.data.get(); leaves.push(leafValue); - sortedLeaves.splice(sortedIndex, 0, leafValue); + sortedLeaves.splice(iSorted, 0, leafValue); + }); + } + + /** + * Append a new leaf based on the pointers of the previous low node + */ + private setLeafUnconstrained( + leafExists: Bool, + low: Leaf, + leaf: BaseLeaf & { index: Field } + ) { + Provable.asProver(() => { + // update internal hash nodes + let i = Number(leaf.index.toBigInt()); + this.setLeafNode(i, Leaf.hashNode(leaf).toBigInt()); + + // update leaf lists + let iSorted = low.sortedIndex.get() + 1; + let leafValue = Leaf.toValue({ + ...leaf, + sortedIndex: Unconstrained.from(iSorted), + }); + + let { leaves, sortedLeaves } = this.data.get(); + + if (leafExists.toBoolean()) { + leaves[i] = leafValue; + sortedLeaves[iSorted] = leafValue; + } else { + leaves.push(leafValue); + sortedLeaves.splice(iSorted, 0, leafValue); + } }); } From 6bc79d82369448ba0863078c0e696341360a04a7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 22:16:31 +0200 Subject: [PATCH 24/93] finish set --- src/lib/provable/merkle-tree-indexed.ts | 81 ++++++++++++++++--------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index b3b3e4383a..8ab88a8cb1 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -1,7 +1,7 @@ import { Poseidon as PoseidonBigint } from '../../bindings/crypto/poseidon.js'; import { Bool, Field } from './wrapped.js'; import { Option } from './option.js'; -import { Struct } from './types/struct.js'; +import { Struct, provableTuple } from './types/struct.js'; import { InferValue } from 'src/bindings/lib/provable-generic.js'; import { assert } from './gadgets/common.js'; import { Unconstrained } from './types/unconstrained.js'; @@ -9,6 +9,7 @@ import { Provable } from './provable.js'; import { Poseidon } from './crypto/poseidon.js'; import { maybeSwap } from './merkle-tree.js'; import { assertDefined } from '../util/errors.js'; +import { provable } from './types/provable-derivers.js'; type IndexedMerkleMapBase = { root: Field; @@ -91,9 +92,20 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { nodes[level] = []; } - this.length = Field(0); - let leaves: LeafValue[] = []; - let sortedLeaves: LeafValue[] = []; + let firstLeaf = { + key: 0n, + value: 0n, + // maximum, which is always greater than any key that is a hash + nextKey: Field.ORDER - 1n, + index: 0n, + nextIndex: 0n, // TODO: ok? + sortedIndex: Unconstrained.from(0), + }; + this.setLeafNode(0, Leaf.hashNode(Leaf.fromValue(firstLeaf)).toBigInt()); + + this.length = Field(1); + let leaves: LeafValue[] = [firstLeaf]; + let sortedLeaves: LeafValue[] = [firstLeaf]; this.data = Unconstrained.from({ leaves, sortedLeaves, nodes }); } @@ -115,9 +127,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { update(key: Field, value: Field) { // prove that the key exists by presenting a leaf that contains it - let self = Provable.witness(Leaf, () => - assertDefined(this.findLeaf(key).self, 'Key does not exist in the tree') - ); + let self = Provable.witness(Leaf, () => this.findLeaf(key).self); this.proveInclusion(self, 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf'); @@ -127,7 +137,10 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { set(key: Field, value: Field) { // prove whether the key exists or not, by showing a valid low node and checking if it points to the key - let low = Provable.witness(Leaf, () => this.findLeaf(key).low); + let { low, self } = Provable.witness( + provable({ low: Leaf, self: Leaf }), + () => this.findLeaf(key) + ); this.proveInclusion(low, 'Invalid low node'); low.key.assertLessThan(key, 'Invalid low node'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node'); @@ -135,19 +148,30 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); + // prove inclusion of this node if it already exists + this.proveInclusionIf(keyExists, self, 'Invalid leaf'); + assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf'); + // the leaf's index depends on whether it exists let index = Provable.if(keyExists, low.nextIndex, this.length); // update low node, or leave it as is let newLow = { ...low, nextKey: key, nextIndex: index }; - let nodeLow = Leaf.hashNode(newLow); - this.root = this.computeRoot(low.index, nodeLow); + this.root = this.computeRoot(low.index, Leaf.hashNode(newLow)); + this.setLeafUnconstrained(true, newLow); // update leaf, or append a new one - // TODO damn I think we need another inclusion proof for the previous leaf - throw Error('not implemented'); - let TODO: any = {}; - this.setLeafUnconstrained(keyExists, low, TODO); + let newLeaf = { + key, + value, + nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), + nextIndex: Provable.if(keyExists, self.nextIndex, low.nextIndex), + index, + sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), + }; + this.root = this.computeRoot(index, Leaf.hashNode(newLeaf)); + this.length = Provable.if(keyExists, this.length, this.length.add(1)); + this.setLeafUnconstrained(keyExists, newLeaf); } get(key: Field): Option { @@ -167,6 +191,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); } + proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { + let node = Leaf.hashNode(leaf); + let root = this.computeRoot(leaf.index, node); + assert( + condition.implies(root.equals(this.root)), + message ?? 'Leaf is not included in the tree' + ); + } + /** * Update existing leaf. * @@ -224,26 +257,18 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { /** * Append a new leaf based on the pointers of the previous low node */ - private setLeafUnconstrained( - leafExists: Bool, - low: Leaf, - leaf: BaseLeaf & { index: Field } - ) { + private setLeafUnconstrained(leafExists: Bool | boolean, leaf: Leaf) { Provable.asProver(() => { // update internal hash nodes let i = Number(leaf.index.toBigInt()); this.setLeafNode(i, Leaf.hashNode(leaf).toBigInt()); // update leaf lists - let iSorted = low.sortedIndex.get() + 1; - let leafValue = Leaf.toValue({ - ...leaf, - sortedIndex: Unconstrained.from(iSorted), - }); - + let leafValue = Leaf.toValue(leaf); + let iSorted = leaf.sortedIndex.get(); let { leaves, sortedLeaves } = this.data.get(); - if (leafExists.toBoolean()) { + if (Bool(leafExists).toBoolean()) { leaves[i] = leafValue; sortedLeaves[iSorted] = leafValue; } else { @@ -296,8 +321,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { leaves.length ); let low = foundValue ? leaves[lowIndex - 1] : leaves[lowIndex]; - let self = foundValue ? leaves[lowIndex] : undefined; - return { low, self }; + let self = foundValue ? leaves[lowIndex] : Leaf.toValue(Leaf.empty()); + return { foundValue, low, self }; } // invariant: for every node that is not undefined, its descendants are either empty or not undefined From 33c216e69b7c57a895d850831f1ad00b52ddde3b Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 22:30:41 +0200 Subject: [PATCH 25/93] give option a constructor --- src/lib/provable/option.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/lib/provable/option.ts b/src/lib/provable/option.ts index 372bcd5f6c..cb9d9b3048 100644 --- a/src/lib/provable/option.ts +++ b/src/lib/provable/option.ts @@ -39,19 +39,23 @@ function Option>( ): Provable< Option, InferValue>, InferValue | undefined -> & { - fromValue( - value: - | { isSome: boolean | Bool; value: InferProvable | InferValue } - | InferProvable - | InferValue - | undefined - ): Option, InferValue>; - from( - value?: InferProvable | InferValue - ): Option, InferValue>; - none(): Option, InferValue>; -} { +> & + (new (option: { isSome: Bool; value: InferProvable }) => Option< + InferProvable, + InferValue + >) & { + fromValue( + value: + | { isSome: boolean | Bool; value: InferProvable | InferValue } + | InferProvable + | InferValue + | undefined + ): Option, InferValue>; + from( + value?: InferProvable | InferValue + ): Option, InferValue>; + none(): Option, InferValue>; + } { type T = InferProvable; type V = InferValue; let strictType: Provable = type; From 910e5748c33a6335927eef82e2c4911b76560122 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 22:30:49 +0200 Subject: [PATCH 26/93] implement get, remove remove --- src/lib/provable/merkle-tree-indexed.ts | 35 +++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 8ab88a8cb1..85c9523afa 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -1,15 +1,13 @@ import { Poseidon as PoseidonBigint } from '../../bindings/crypto/poseidon.js'; import { Bool, Field } from './wrapped.js'; import { Option } from './option.js'; -import { Struct, provableTuple } from './types/struct.js'; +import { Struct } from './types/struct.js'; import { InferValue } from 'src/bindings/lib/provable-generic.js'; import { assert } from './gadgets/common.js'; import { Unconstrained } from './types/unconstrained.js'; import { Provable } from './provable.js'; import { Poseidon } from './crypto/poseidon.js'; import { maybeSwap } from './merkle-tree.js'; -import { assertDefined } from '../util/errors.js'; -import { provable } from './types/provable-derivers.js'; type IndexedMerkleMapBase = { root: Field; @@ -29,7 +27,7 @@ type IndexedMerkleMapBase = { // optional / nice-to-have: remove a key and its value from the tree; proves that the key is included. // (implementation: leave a wasted leaf in place but skip it in the linked list encoding) - remove(key: Field): void; + // remove(key: Field): void; }; type BaseLeaf = { @@ -56,6 +54,9 @@ class Leaf extends Struct({ } type LeafValue = InferValue; +class OptionField extends Option(Field) {} +class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} + class IndexedMerkleMap implements IndexedMerkleMapBase { // data defining the provable interface of a tree root: Field; @@ -136,11 +137,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } set(key: Field, value: Field) { - // prove whether the key exists or not, by showing a valid low node and checking if it points to the key - let { low, self } = Provable.witness( - provable({ low: Leaf, self: Leaf }), - () => this.findLeaf(key) - ); + // prove whether the key exists or not, by showing a valid low node + let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); this.proveInclusion(low, 'Invalid low node'); low.key.assertLessThan(key, 'Invalid low node'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node'); @@ -148,7 +146,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); - // prove inclusion of this node if it already exists + // prove inclusion of this leaf if it exists this.proveInclusionIf(keyExists, self, 'Invalid leaf'); assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf'); @@ -175,11 +173,20 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } get(key: Field): Option { - assert(false, 'not implemented'); - } + // prove whether the key exists or not, by showing a valid low node + let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); + this.proveInclusion(low, 'Invalid low node'); + low.key.assertLessThan(key, 'Invalid low node'); + key.assertLessThanOrEqual(low.nextKey, 'Invalid low node'); + + // the key exists iff lowNode.nextKey == key + let keyExists = low.nextKey.equals(key); + + // prove inclusion of this leaf if it exists + this.proveInclusionIf(keyExists, self, 'Invalid leaf'); + assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf'); - remove(key: Field) { - assert(false, 'not implemented'); + return new OptionField({ isSome: keyExists, value: self.value }); } // helper methods From 97f03acadfb34c583b377aa41ae71d96f069e692 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 22:37:32 +0200 Subject: [PATCH 27/93] simplify --- src/lib/provable/merkle-tree-indexed.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 85c9523afa..38c160d932 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -65,16 +65,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // the raw data stored in the tree, plus helper structures readonly data: Unconstrained<{ - // leaves sorted by index - readonly leaves: LeafValue[]; + // for every level, an array of hashes + readonly nodes: (bigint | undefined)[][]; // leaves sorted by key, with a circular linked list encoded by nextKey - // we always have sortedLeaves[n-1].nextKey = 0 = sortedLeaves[0].key + // we always have + // sortedLeaves[0].key = 0 + // sortedLeaves[n-1].nextKey = Field.ORDER - 1 // for i=0,...n-2, sortedLeaves[i].nextKey = sortedLeaves[i+1].key readonly sortedLeaves: LeafValue[]; - - // for every level, an array of hashes - readonly nodes: (bigint | undefined)[][]; }>; /** @@ -224,9 +223,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafNode(i, node.toBigInt()); // update leaf lists - let { leaves, sortedLeaves } = this.data.get(); + let { sortedLeaves } = this.data.get(); let leafValue = Leaf.toValue(newLeaf); - leaves[i] = leafValue; sortedLeaves[leaf.sortedIndex.get()] = leafValue; }); } @@ -255,8 +253,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { sortedIndex: Unconstrained.from(iSorted), }); - let { leaves, sortedLeaves } = this.data.get(); - leaves.push(leafValue); + let { sortedLeaves } = this.data.get(); sortedLeaves.splice(iSorted, 0, leafValue); }); } @@ -273,13 +270,11 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // update leaf lists let leafValue = Leaf.toValue(leaf); let iSorted = leaf.sortedIndex.get(); - let { leaves, sortedLeaves } = this.data.get(); + let { sortedLeaves } = this.data.get(); if (Bool(leafExists).toBoolean()) { - leaves[i] = leafValue; sortedLeaves[iSorted] = leafValue; } else { - leaves.push(leafValue); sortedLeaves.splice(iSorted, 0, leafValue); } }); From 23eede482c39d9a9ce49b7d4c544db3958bfa875 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 22:37:48 +0200 Subject: [PATCH 28/93] correct initial root --- src/lib/provable/merkle-tree-indexed.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 38c160d932..728ce499b5 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -85,7 +85,6 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { height < 53, 'height must be less than 53, so that we can use 64-bit floats to represent indices.' ); - this.root = Field(empty(height - 1)); let nodes: (bigint | undefined)[][] = Array(height); for (let level = 0; level < height; level++) { @@ -101,7 +100,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { nextIndex: 0n, // TODO: ok? sortedIndex: Unconstrained.from(0), }; - this.setLeafNode(0, Leaf.hashNode(Leaf.fromValue(firstLeaf)).toBigInt()); + let firstNode = Leaf.hashNode(Leaf.fromValue(firstLeaf)); + this.setLeafNode(0, firstNode.toBigInt()); + this.root = Field(this.getNode(height - 1, 0, true)); this.length = Field(1); let leaves: LeafValue[] = [firstLeaf]; From 7201a2c47b6b587ce3a2c27390f355b188d27c30 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 28 May 2024 22:40:13 +0200 Subject: [PATCH 29/93] better name --- src/lib/provable/merkle-tree-indexed.ts | 6 ++++-- src/lib/provable/merkle-tree.ts | 6 +++--- src/lib/provable/test/merkle-tree.unit-test.ts | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 728ce499b5..485b7ab494 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -7,7 +7,9 @@ import { assert } from './gadgets/common.js'; import { Unconstrained } from './types/unconstrained.js'; import { Provable } from './provable.js'; import { Poseidon } from './crypto/poseidon.js'; -import { maybeSwap } from './merkle-tree.js'; +import { conditionalSwap } from './merkle-tree.js'; + +export { IndexedMerkleMap }; type IndexedMerkleMapBase = { root: Field; @@ -296,7 +298,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { let isLeft = !isRight.toBoolean(); return this.getNode(level, isLeft ? i + 1 : i - 1, false); }); - let [right, left] = maybeSwap(isRight, node, sibling); + let [right, left] = conditionalSwap(isRight, node, sibling); node = Poseidon.hash([left, right]); indexU.updateAsProver((i) => i >> 1); } diff --git a/src/lib/provable/merkle-tree.ts b/src/lib/provable/merkle-tree.ts index a1be3b0a91..1a102cb9c5 100644 --- a/src/lib/provable/merkle-tree.ts +++ b/src/lib/provable/merkle-tree.ts @@ -11,7 +11,7 @@ import { Provable } from './provable.js'; export { Witness, MerkleTree, MerkleWitness, BaseMerkleWitness }; // internal API -export { maybeSwap }; +export { conditionalSwap }; type Witness = { isLeft: boolean; sibling: Field }[]; @@ -207,7 +207,7 @@ class BaseMerkleWitness extends CircuitValue { for (let i = 1; i < n; ++i) { let isLeft = this.isLeft[i - 1]; - const [left, right] = maybeSwap(isLeft, hash, this.path[i - 1]); + const [left, right] = conditionalSwap(isLeft, hash, this.path[i - 1]); hash = Poseidon.hash([left, right]); } @@ -248,7 +248,7 @@ function MerkleWitness(height: number): typeof BaseMerkleWitness { // swap two values if the boolean is false, otherwise keep them as they are // more efficient than 2x `Provable.if()` by reusing an intermediate variable -function maybeSwap(b: Bool, x: Field, y: Field): [Field, Field] { +function conditionalSwap(b: Bool, x: Field, y: Field): [Field, Field] { let m = b.toField().mul(x.sub(y)); // b*(x - y) const x_ = y.add(m); // y + b*(x - y) const y_ = x.sub(m); // x - b*(x - y) = x + b*(y - x) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 6e1cd887e6..bd4aa04464 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -1,11 +1,11 @@ import { Bool, Field } from '../wrapped.js'; -import { maybeSwap } from '../merkle-tree.js'; +import { conditionalSwap } from '../merkle-tree.js'; import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; import { MerkleMap } from '../merkle-map.js'; test(Random.bool, Random.field, Random.field, (b, x, y) => { - let [x0, y0] = maybeSwap(Bool(!!b), Field(x), Field(y)); + let [x0, y0] = conditionalSwap(Bool(!!b), Field(x), Field(y)); // if the boolean is true, it shouldn't swap the fields; otherwise, it should if (b) { From 8bb2290c1dc1beba5d773497464a6aa9125d0dff Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 09:28:52 +0200 Subject: [PATCH 30/93] simplify --- src/lib/provable/merkle-tree-indexed.ts | 146 ++++++++++-------------- 1 file changed, 59 insertions(+), 87 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 485b7ab494..4396037516 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -53,6 +53,17 @@ class Leaf extends Struct({ static hashNode({ key, value, nextKey, nextIndex }: BaseLeaf) { return Poseidon.hash([key, value, nextKey, nextIndex]); } + + static nextAfter(low: Leaf, leaf: BaseLeaf): Leaf { + return { + key: leaf.key, + value: leaf.value, + nextKey: leaf.key, + nextIndex: leaf.nextIndex, + index: low.nextIndex, + sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), + }; + } } type LeafValue = InferValue; @@ -107,9 +118,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.root = Field(this.getNode(height - 1, 0, true)); this.length = Field(1); - let leaves: LeafValue[] = [firstLeaf]; let sortedLeaves: LeafValue[] = [firstLeaf]; - this.data = Unconstrained.from({ leaves, sortedLeaves, nodes }); + this.data = Unconstrained.from({ nodes, sortedLeaves }); } insert(key: Field, value: Field) { @@ -122,10 +132,21 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { key.assertLessThan(low.nextKey, 'Key already exists in the tree'); // update low node - this.updateLeaf(low, { nextKey: key, nextIndex: this.length }); + let index = this.length; + let newLow = { ...low, nextKey: key, nextIndex: index }; + this.root = this.computeRoot(newLow.index, Leaf.hashNode(newLow)); + this.setLeafUnconstrained(true, newLow); - // append new leaf - this.appendLeaf(low, { key, value }); + // create and append new leaf + let leaf = Leaf.nextAfter(newLow, { + key, + value, + nextKey: low.nextKey, + nextIndex: low.nextIndex, + }); + this.root = this.computeRoot(index, Leaf.hashNode(leaf)); + this.length = this.length.add(1); + this.setLeafUnconstrained(false, leaf); } update(key: Field, value: Field) { @@ -135,7 +156,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { self.key.assertEquals(key, 'Invalid leaf'); // update leaf - this.updateLeaf(self, { value }); + let newSelf = { ...self, value }; + this.root = this.computeRoot(self.index, Leaf.hashNode(newSelf)); + this.setLeafUnconstrained(true, newSelf); } set(key: Field, value: Field) { @@ -161,14 +184,12 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(true, newLow); // update leaf, or append a new one - let newLeaf = { + let newLeaf = Leaf.nextAfter(newLow, { key, value, nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), nextIndex: Provable.if(keyExists, self.nextIndex, low.nextIndex), - index, - sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), - }; + }); this.root = this.computeRoot(index, Leaf.hashNode(newLeaf)); this.length = Provable.if(keyExists, this.length, this.length.add(1)); this.setLeafUnconstrained(keyExists, newLeaf); @@ -194,7 +215,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods proveInclusion(leaf: Leaf, message?: string) { - // TODO: here, we don't actually care about the index, so we could add a mode where `computeRoot()` doesn't prove it + // TODO: here, we don't actually care about the index, so we could add a mode where + // `computeRoot()` doesn't prove it let node = Leaf.hashNode(leaf); let root = this.computeRoot(leaf.index, node); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); @@ -209,80 +231,6 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { ); } - /** - * Update existing leaf. - * - * Note: we never update the key or index of a leaf. - */ - updateLeaf(leaf: Leaf, partialLeaf: Partial) { - // update root - let newLeaf = { ...leaf, ...partialLeaf }; - let node = Leaf.hashNode(newLeaf); - this.root = this.computeRoot(leaf.index, node); - - Provable.asProver(() => { - // update internal hash nodes - let i = Number(leaf.index.toBigInt()); - this.setLeafNode(i, node.toBigInt()); - - // update leaf lists - let { sortedLeaves } = this.data.get(); - let leafValue = Leaf.toValue(newLeaf); - sortedLeaves[leaf.sortedIndex.get()] = leafValue; - }); - } - - /** - * Append a new leaf based on the pointers of the previous low node - */ - appendLeaf(low: Leaf, { key, value }: { key: Field; value: Field }) { - // update root and length - let index = this.length; - let leaf = { key, value, nextKey: low.nextKey, nextIndex: low.nextIndex }; - let node = Leaf.hashNode(leaf); - this.root = this.computeRoot(index, node); - this.length = this.length.add(1); - - Provable.asProver(() => { - // update internal hash nodes - let i = Number(index.toBigInt()); - this.setLeafNode(i, node.toBigInt()); - - // update leaf lists - let iSorted = low.sortedIndex.get() + 1; - let leafValue = Leaf.toValue({ - ...leaf, - index, - sortedIndex: Unconstrained.from(iSorted), - }); - - let { sortedLeaves } = this.data.get(); - sortedLeaves.splice(iSorted, 0, leafValue); - }); - } - - /** - * Append a new leaf based on the pointers of the previous low node - */ - private setLeafUnconstrained(leafExists: Bool | boolean, leaf: Leaf) { - Provable.asProver(() => { - // update internal hash nodes - let i = Number(leaf.index.toBigInt()); - this.setLeafNode(i, Leaf.hashNode(leaf).toBigInt()); - - // update leaf lists - let leafValue = Leaf.toValue(leaf); - let iSorted = leaf.sortedIndex.get(); - let { sortedLeaves } = this.data.get(); - - if (Bool(leafExists).toBoolean()) { - sortedLeaves[iSorted] = leafValue; - } else { - sortedLeaves.splice(iSorted, 0, leafValue); - } - }); - } - /** * Compute the root given a leaf node and its index. */ @@ -307,7 +255,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } /** - * Given a key, returns both the low node and the node that contains the key. + * Given a key, returns both the low node and the leaf that contains the key. + * + * If the key does not exist, a dummy value is returned for the leaf. * * Assumes to run outside provable code. */ @@ -318,7 +268,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // this case is typically invalid, but we want to handle it gracefully here // and reject it using comparison constraints - if (key === 0n) return { low: leaves[leaves.length - 1], self: leaves[0] }; + if (key === 0n) return { low: Leaf.toValue(Leaf.empty()), self: leaves[0] }; let { lowIndex, foundValue } = bisectUnique( key, @@ -330,6 +280,28 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { return { foundValue, low, self }; } + /** + * Update or append a leaf in our internal data structures + */ + private setLeafUnconstrained(leafExists: Bool | boolean, leaf: Leaf) { + Provable.asProver(() => { + // update internal hash nodes + let i = Number(leaf.index.toBigInt()); + this.setLeafNode(i, Leaf.hashNode(leaf).toBigInt()); + + // update sorted list + let leafValue = Leaf.toValue(leaf); + let iSorted = leaf.sortedIndex.get(); + let { sortedLeaves } = this.data.get(); + + if (Bool(leafExists).toBoolean()) { + sortedLeaves[iSorted] = leafValue; + } else { + sortedLeaves.splice(iSorted, 0, leafValue); + } + }); + } + // invariant: for every node that is not undefined, its descendants are either empty or not undefined private setLeafNode(index: number, leaf: bigint) { let nodes = this.data.get().nodes; From 258a64eb9f4d875a3346a4a0764875a19dd7c6fc Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 09:32:40 +0200 Subject: [PATCH 31/93] api friendliness --- src/lib/provable/merkle-tree-indexed.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 4396037516..3e59e8bd63 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -122,7 +122,10 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.data = Unconstrained.from({ nodes, sortedLeaves }); } - insert(key: Field, value: Field) { + insert(key: Field | bigint, value: Field | bigint) { + key = Field(key); + value = Field(value); + // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this.findLeaf(key).low); this.proveInclusion(low, 'Invalid low node'); @@ -149,7 +152,10 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(false, leaf); } - update(key: Field, value: Field) { + update(key: Field | bigint, value: Field | bigint) { + key = Field(key); + value = Field(value); + // prove that the key exists by presenting a leaf that contains it let self = Provable.witness(Leaf, () => this.findLeaf(key).self); this.proveInclusion(self, 'Key does not exist in the tree'); @@ -161,7 +167,10 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(true, newSelf); } - set(key: Field, value: Field) { + set(key: Field | bigint, value: Field | bigint) { + key = Field(key); + value = Field(value); + // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); this.proveInclusion(low, 'Invalid low node'); @@ -195,7 +204,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(keyExists, newLeaf); } - get(key: Field): Option { + get(key: Field | bigint): Option { + key = Field(key); + // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); this.proveInclusion(low, 'Invalid low node'); From 5028c39894f71087182a44bcc05bc0b89cd52bd3 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 09:50:16 +0200 Subject: [PATCH 32/93] fix constructor --- src/lib/provable/merkle-tree-indexed.ts | 58 +++++++++++-------- .../provable/test/merkle-tree.unit-test.ts | 6 ++ 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 3e59e8bd63..a370017b4e 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -114,12 +114,11 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { sortedIndex: Unconstrained.from(0), }; let firstNode = Leaf.hashNode(Leaf.fromValue(firstLeaf)); - this.setLeafNode(0, firstNode.toBigInt()); - this.root = Field(this.getNode(height - 1, 0, true)); - + let root = Nodes.setLeafNode(nodes, 0, firstNode.toBigInt()); + this.root = Field(root); this.length = Field(1); - let sortedLeaves: LeafValue[] = [firstLeaf]; - this.data = Unconstrained.from({ nodes, sortedLeaves }); + + this.data = Unconstrained.from({ nodes, sortedLeaves: [firstLeaf] }); } insert(key: Field | bigint, value: Field | bigint) { @@ -255,7 +254,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { let sibling = Provable.witness(Field, () => { let i = indexU.get(); let isLeft = !isRight.toBoolean(); - return this.getNode(level, isLeft ? i + 1 : i - 1, false); + let nodes = this.data.get().nodes; + return Nodes.getNode(nodes, level, isLeft ? i + 1 : i - 1, false); }); let [right, left] = conditionalSwap(isRight, node, sibling); node = Poseidon.hash([left, right]); @@ -298,7 +298,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { Provable.asProver(() => { // update internal hash nodes let i = Number(leaf.index.toBigInt()); - this.setLeafNode(i, Leaf.hashNode(leaf).toBigInt()); + let nodes = this.data.get().nodes; + Nodes.setLeafNode(nodes, i, Leaf.hashNode(leaf).toBigInt()); // update sorted list let leafValue = Leaf.toValue(leaf); @@ -312,27 +313,36 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } }); } +} - // invariant: for every node that is not undefined, its descendants are either empty or not undefined - private setLeafNode(index: number, leaf: bigint) { - let nodes = this.data.get().nodes; - +type Nodes = (bigint | undefined)[][]; +namespace Nodes { + /** + * Sets the leaf node at the given index, updates all parent nodes and returns the new root. + */ + export function setLeafNode(nodes: Nodes, index: number, leaf: bigint) { nodes[0][index] = leaf; let isLeft = index % 2 === 0; + let height = nodes.length; - for (let level = 1; level < this.height; level++) { + for (let level = 1; level < height; level++) { index = Math.floor(index / 2); - let left = this.getNode(level - 1, index * 2, isLeft); - let right = this.getNode(level - 1, index * 2 + 1, !isLeft); + let left = getNode(nodes, level - 1, index * 2, isLeft); + let right = getNode(nodes, level - 1, index * 2 + 1, !isLeft); nodes[level][index] = PoseidonBigint.hash([left, right]); isLeft = index % 2 === 0; } + return getNode(nodes, height - 1, 0, true); } - private getNode(level: number, index: number, nonEmpty: boolean) { - let nodes = this.data.get().nodes; + export function getNode( + nodes: Nodes, + level: number, + index: number, + nonEmpty: boolean + ) { let node = nodes[level]?.[index]; if (node === undefined) { if (nonEmpty) @@ -343,17 +353,17 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } return node; } -} -// cache of empty nodes (=: zero leaves and nodes with only empty nodes below them) -const emptyNodes = [0n]; + // cache of empty nodes (=: zero leaves and nodes with only empty nodes below them) + const emptyNodes = [0n]; -function empty(level: number) { - for (let i = emptyNodes.length; i <= level; i++) { - let zero = emptyNodes[i - 1]; - emptyNodes[i] = PoseidonBigint.hash([zero, zero]); + export function empty(level: number) { + for (let i = emptyNodes.length; i <= level; i++) { + let zero = emptyNodes[i - 1]; + emptyNodes[i] = PoseidonBigint.hash([zero, zero]); + } + return emptyNodes[level]; } - return emptyNodes[level]; } // helper diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index bd4aa04464..90e584f47b 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -3,6 +3,12 @@ import { conditionalSwap } from '../merkle-tree.js'; import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; import { MerkleMap } from '../merkle-map.js'; +import { IndexedMerkleMap } from '../merkle-tree-indexed.js'; +import { Provable } from '../provable.js'; + +let map = new IndexedMerkleMap(32); +let x = map.get(0n); +Provable.log('value at 0', x); test(Random.bool, Random.field, Random.field, (b, x, y) => { let [x0, y0] = conditionalSwap(Bool(!!b), Field(x), Field(y)); From a2efac847e08ec9943e14d7b9f1e887c779cac98 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 09:51:05 +0200 Subject: [PATCH 33/93] minor --- src/lib/provable/merkle-tree-indexed.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index a370017b4e..a854277db6 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -296,15 +296,15 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { */ private setLeafUnconstrained(leafExists: Bool | boolean, leaf: Leaf) { Provable.asProver(() => { + let { nodes, sortedLeaves } = this.data.get(); + // update internal hash nodes let i = Number(leaf.index.toBigInt()); - let nodes = this.data.get().nodes; Nodes.setLeafNode(nodes, i, Leaf.hashNode(leaf).toBigInt()); // update sorted list let leafValue = Leaf.toValue(leaf); let iSorted = leaf.sortedIndex.get(); - let { sortedLeaves } = this.data.get(); if (Bool(leafExists).toBoolean()) { sortedLeaves[iSorted] = leafValue; From 42438271825498d1c7707f8db3d51380fe9d1f66 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 11:02:05 +0200 Subject: [PATCH 34/93] fix several bugs --- src/lib/provable/merkle-tree-indexed.ts | 106 ++++++++++++------ .../provable/test/merkle-tree.unit-test.ts | 26 ++++- 2 files changed, 92 insertions(+), 40 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index a854277db6..c9d6ce6276 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -58,14 +58,33 @@ class Leaf extends Struct({ return { key: leaf.key, value: leaf.value, - nextKey: leaf.key, + nextKey: leaf.nextKey, nextIndex: leaf.nextIndex, index: low.nextIndex, sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), }; } + + static toBigints(leaf: Leaf): LeafValue { + return { + key: leaf.key.toBigInt(), + value: leaf.value.toBigInt(), + nextKey: leaf.nextKey.toBigInt(), + index: leaf.index.toBigInt(), + nextIndex: leaf.nextIndex.toBigInt(), + }; + } } -type LeafValue = InferValue; + +type LeafValue = { + value: bigint; + + key: bigint; + nextKey: bigint; + + index: bigint; + nextIndex: bigint; +}; class OptionField extends Option(Field) {} class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} @@ -81,7 +100,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // for every level, an array of hashes readonly nodes: (bigint | undefined)[][]; - // leaves sorted by key, with a circular linked list encoded by nextKey + // leaves sorted by key, with a linked list encoded by nextKey // we always have // sortedLeaves[0].key = 0 // sortedLeaves[n-1].nextKey = Field.ORDER - 1 @@ -98,6 +117,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { height < 53, 'height must be less than 53, so that we can use 64-bit floats to represent indices.' ); + this.height = height; let nodes: (bigint | undefined)[][] = Array(height); for (let level = 0; level < height; level++) { @@ -111,9 +131,10 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { nextKey: Field.ORDER - 1n, index: 0n, nextIndex: 0n, // TODO: ok? - sortedIndex: Unconstrained.from(0), }; - let firstNode = Leaf.hashNode(Leaf.fromValue(firstLeaf)); + let firstNode = Leaf.hashNode( + Leaf.fromValue({ ...firstLeaf, sortedIndex: Unconstrained.from(0) }) + ); let root = Nodes.setLeafNode(nodes, 0, firstNode.toBigInt()); this.root = Field(root); this.length = Field(1); @@ -127,8 +148,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this.findLeaf(key).low); - this.proveInclusion(low, 'Invalid low node'); - low.key.assertLessThan(key, 'Invalid low node'); + this.proveInclusion(low, 'Invalid low node (root)'); + low.key.assertLessThan(key, 'Invalid low node (key)'); // if the key does exist, we have lowNode.nextKey == key, and this line fails key.assertLessThan(low.nextKey, 'Key already exists in the tree'); @@ -146,6 +167,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { nextKey: low.nextKey, nextIndex: low.nextIndex, }); + this.root = this.computeRoot(index, Leaf.hashNode(leaf)); this.length = this.length.add(1); this.setLeafUnconstrained(false, leaf); @@ -158,7 +180,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // prove that the key exists by presenting a leaf that contains it let self = Provable.witness(Leaf, () => this.findLeaf(key).self); this.proveInclusion(self, 'Key does not exist in the tree'); - self.key.assertEquals(key, 'Invalid leaf'); + self.key.assertEquals(key, 'Invalid leaf (key)'); // update leaf let newSelf = { ...self, value }; @@ -172,16 +194,16 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); - this.proveInclusion(low, 'Invalid low node'); - low.key.assertLessThan(key, 'Invalid low node'); - key.assertLessThanOrEqual(low.nextKey, 'Invalid low node'); + this.proveInclusion(low, 'Invalid low node (root)'); + low.key.assertLessThan(key, 'Invalid low node (key)'); + key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); // prove inclusion of this leaf if it exists - this.proveInclusionIf(keyExists, self, 'Invalid leaf'); - assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf'); + this.proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); + assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); // the leaf's index depends on whether it exists let index = Provable.if(keyExists, low.nextIndex, this.length); @@ -208,16 +230,16 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); - this.proveInclusion(low, 'Invalid low node'); - low.key.assertLessThan(key, 'Invalid low node'); - key.assertLessThanOrEqual(low.nextKey, 'Invalid low node'); + this.proveInclusion(low, 'Invalid low node (root)'); + low.key.assertLessThan(key, 'Invalid low node (key)'); + key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); // prove inclusion of this leaf if it exists - this.proveInclusionIf(keyExists, self, 'Invalid leaf'); - assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf'); + this.proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); + assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); return new OptionField({ isSome: keyExists, value: self.value }); } @@ -225,8 +247,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods proveInclusion(leaf: Leaf, message?: string) { - // TODO: here, we don't actually care about the index, so we could add a mode where - // `computeRoot()` doesn't prove it + // TODO: here, we don't actually care about the index, + // so we could add a mode where `computeRoot()` doesn't prove it let node = Leaf.hashNode(leaf); let root = this.computeRoot(leaf.index, node); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); @@ -245,7 +267,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { * Compute the root given a leaf node and its index. */ computeRoot(index: Field, node: Field) { - let indexU = Unconstrained.from(Number(index.toBigInt())); + let indexU = Unconstrained.witness(() => Number(index.toBigInt())); let indexBits = index.toBits(this.height - 1); for (let level = 0; level < this.height - 1; level++) { @@ -253,13 +275,19 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { let isRight = indexBits[level]; let sibling = Provable.witness(Field, () => { let i = indexU.get(); + indexU.set(i >> 1); let isLeft = !isRight.toBoolean(); let nodes = this.data.get().nodes; - return Nodes.getNode(nodes, level, isLeft ? i + 1 : i - 1, false); + let sibling = Nodes.getNode( + nodes, + level, + isLeft ? i + 1 : i - 1, + false + ); + return sibling; }); let [right, left] = conditionalSwap(isRight, node, sibling); node = Poseidon.hash([left, right]); - indexU.updateAsProver((i) => i >> 1); } // now, `node` is the root of the tree return node; @@ -272,23 +300,31 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { * * Assumes to run outside provable code. */ - private findLeaf(key_: Field | bigint) { + findLeaf(key_: Field | bigint): InferValue { let key = typeof key_ === 'bigint' ? key_ : key_.toBigInt(); assert(key >= 0n, 'key must be positive'); let leaves = this.data.get().sortedLeaves; // this case is typically invalid, but we want to handle it gracefully here // and reject it using comparison constraints - if (key === 0n) return { low: Leaf.toValue(Leaf.empty()), self: leaves[0] }; + if (key === 0n) + return { + low: Leaf.toValue(Leaf.empty()), + self: { ...leaves[0], sortedIndex: Unconstrained.from(0) }, + }; let { lowIndex, foundValue } = bisectUnique( key, (i) => leaves[i].key, leaves.length ); - let low = foundValue ? leaves[lowIndex - 1] : leaves[lowIndex]; - let self = foundValue ? leaves[lowIndex] : Leaf.toValue(Leaf.empty()); - return { foundValue, low, self }; + let iLow = foundValue ? lowIndex - 1 : lowIndex; + let low = { ...leaves[iLow], sortedIndex: Unconstrained.from(iLow) }; + + let iSelf = foundValue ? lowIndex : 0; + let selfBase = foundValue ? leaves[lowIndex] : Leaf.toBigints(Leaf.empty()); + let self = { ...selfBase, sortedIndex: Unconstrained.from(iSelf) }; + return { low, self }; } /** @@ -303,7 +339,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { Nodes.setLeafNode(nodes, i, Leaf.hashNode(leaf).toBigInt()); // update sorted list - let leafValue = Leaf.toValue(leaf); + let leafValue = Leaf.toBigints(leaf); let iSorted = leaf.sortedIndex.get(); if (Bool(leafExists).toBoolean()) { @@ -322,17 +358,15 @@ namespace Nodes { */ export function setLeafNode(nodes: Nodes, index: number, leaf: bigint) { nodes[0][index] = leaf; - let isLeft = index % 2 === 0; let height = nodes.length; - for (let level = 1; level < height; level++) { + for (let level = 0; level < height - 1; level++) { + let isLeft = index % 2 === 0; index = Math.floor(index / 2); - let left = getNode(nodes, level - 1, index * 2, isLeft); - let right = getNode(nodes, level - 1, index * 2 + 1, !isLeft); - nodes[level][index] = PoseidonBigint.hash([left, right]); - - isLeft = index % 2 === 0; + let left = getNode(nodes, level, index * 2, isLeft); + let right = getNode(nodes, level, index * 2 + 1, !isLeft); + nodes[level + 1][index] = PoseidonBigint.hash([left, right]); } return getNode(nodes, height - 1, 0, true); } diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 90e584f47b..5c8778d546 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -4,11 +4,29 @@ import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; import { MerkleMap } from '../merkle-map.js'; import { IndexedMerkleMap } from '../merkle-tree-indexed.js'; -import { Provable } from '../provable.js'; -let map = new IndexedMerkleMap(32); -let x = map.get(0n); -Provable.log('value at 0', x); +console.log('new map'); +let map = new IndexedMerkleMap(3); + +console.log('insert 2 and 1'); +map.insert(2n, 14n); +map.insert(1n, 13n); +// console.dir(map.findLeaf(2n), { depth: null }); + +console.log('test'); +expect(map.get(1n).assertSome().toBigInt()).toEqual(13n); +expect(map.get(2n).assertSome().toBigInt()).toEqual(14n); + +console.dir(map.data.get().sortedLeaves, { depth: null }); + +console.log('update 2 and 0'); +map.update(2n, 15n); +map.update(0n, 12n); +// TODO get() doesn't work on 0n because the low node checks fail +// expect(map.get(0n).assertSome().toBigInt()).toEqual(12n); +expect(map.get(2n).assertSome().toBigInt()).toEqual(15n); + +console.dir(map.data.get().sortedLeaves, { depth: null }); test(Random.bool, Random.field, Random.field, (b, x, y) => { let [x0, y0] = conditionalSwap(Bool(!!b), Field(x), Field(y)); From 70da09778bfa2ac93ed44122fef422d940168dbc Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 11:27:24 +0200 Subject: [PATCH 35/93] fix provable from class --- src/lib/provable/bytes.ts | 1 + src/lib/provable/crypto/foreign-curve.ts | 1 + src/lib/provable/crypto/foreign-ecdsa.ts | 1 + src/lib/provable/types/provable-derivers.ts | 6 +++--- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/provable/bytes.ts b/src/lib/provable/bytes.ts index cfcd93fae5..645a2d7f6e 100644 --- a/src/lib/provable/bytes.ts +++ b/src/lib/provable/bytes.ts @@ -198,6 +198,7 @@ class Bytes { static _size?: number; static _provable?: ProvablePureExtended< Bytes, + { bytes: { value: bigint }[] }, { bytes: { value: string }[] } >; diff --git a/src/lib/provable/crypto/foreign-curve.ts b/src/lib/provable/crypto/foreign-curve.ts index a47976bba1..0bdfdb016b 100644 --- a/src/lib/provable/crypto/foreign-curve.ts +++ b/src/lib/provable/crypto/foreign-curve.ts @@ -242,6 +242,7 @@ class ForeignCurve { static _Scalar?: typeof AlmostForeignField; static _provable?: ProvablePureExtended< ForeignCurve, + { x: bigint; y: bigint }, { x: string; y: string } >; diff --git a/src/lib/provable/crypto/foreign-ecdsa.ts b/src/lib/provable/crypto/foreign-ecdsa.ts index b601928b93..03f9e4029e 100644 --- a/src/lib/provable/crypto/foreign-ecdsa.ts +++ b/src/lib/provable/crypto/foreign-ecdsa.ts @@ -164,6 +164,7 @@ class EcdsaSignature { static _Curve?: typeof ForeignCurve; static _provable?: ProvablePureExtended< EcdsaSignature, + { r: bigint; s: bigint }, { r: string; s: string } >; diff --git a/src/lib/provable/types/provable-derivers.ts b/src/lib/provable/types/provable-derivers.ts index 7126b11ec1..b417b23163 100644 --- a/src/lib/provable/types/provable-derivers.ts +++ b/src/lib/provable/types/provable-derivers.ts @@ -58,7 +58,7 @@ const { provable } = createDerivers(); function provablePure( typeObj: A -): ProvablePureExtended, InferJson> { +): ProvablePureExtended, InferValue, InferJson> { return provable(typeObj, { isPure: true }) as any; } @@ -70,8 +70,8 @@ function provableFromClass>( Class: Constructor & { check?: (x: T) => void; empty?: () => T }, typeObj: A ): IsPure extends true - ? ProvablePureExtended> - : ProvableExtended> { + ? ProvablePureExtended, InferJson> + : ProvableExtended, InferJson> { let raw = provable(typeObj); return { sizeInFields: raw.sizeInFields, From 3aac569cd6c3e1e8a8c4924f4779212c0eeb6dab Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 11:33:59 +0200 Subject: [PATCH 36/93] move around code --- src/lib/provable/merkle-tree-indexed.ts | 119 ++++++++++++------------ 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index c9d6ce6276..01eb052c57 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -8,6 +8,7 @@ import { Unconstrained } from './types/unconstrained.js'; import { Provable } from './provable.js'; import { Poseidon } from './crypto/poseidon.js'; import { conditionalSwap } from './merkle-tree.js'; +import { provableFromClass } from './types/provable-derivers.js'; export { IndexedMerkleMap }; @@ -32,63 +33,6 @@ type IndexedMerkleMapBase = { // remove(key: Field): void; }; -type BaseLeaf = { - key: Field; - value: Field; - nextKey: Field; - nextIndex: Field; -}; - -class Leaf extends Struct({ - value: Field, - - key: Field, - nextKey: Field, - - index: Field, - nextIndex: Field, - - sortedIndex: Unconstrained.provableWithEmpty(0), -}) { - static hashNode({ key, value, nextKey, nextIndex }: BaseLeaf) { - return Poseidon.hash([key, value, nextKey, nextIndex]); - } - - static nextAfter(low: Leaf, leaf: BaseLeaf): Leaf { - return { - key: leaf.key, - value: leaf.value, - nextKey: leaf.nextKey, - nextIndex: leaf.nextIndex, - index: low.nextIndex, - sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), - }; - } - - static toBigints(leaf: Leaf): LeafValue { - return { - key: leaf.key.toBigInt(), - value: leaf.value.toBigInt(), - nextKey: leaf.nextKey.toBigInt(), - index: leaf.index.toBigInt(), - nextIndex: leaf.nextIndex.toBigInt(), - }; - } -} - -type LeafValue = { - value: bigint; - - key: bigint; - nextKey: bigint; - - index: bigint; - nextIndex: bigint; -}; - -class OptionField extends Option(Field) {} -class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} - class IndexedMerkleMap implements IndexedMerkleMapBase { // data defining the provable interface of a tree root: Field; @@ -351,6 +295,8 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } } +// helpers for updating nodes + type Nodes = (bigint | undefined)[][]; namespace Nodes { /** @@ -400,6 +346,65 @@ namespace Nodes { } } +// leaf + +type BaseLeaf = { + key: Field; + value: Field; + nextKey: Field; + nextIndex: Field; +}; + +class Leaf extends Struct({ + value: Field, + + key: Field, + nextKey: Field, + + index: Field, + nextIndex: Field, + + sortedIndex: Unconstrained.provableWithEmpty(0), +}) { + static hashNode({ key, value, nextKey, nextIndex }: BaseLeaf) { + return Poseidon.hash([key, value, nextKey, nextIndex]); + } + + static nextAfter(low: Leaf, leaf: BaseLeaf): Leaf { + return { + key: leaf.key, + value: leaf.value, + nextKey: leaf.nextKey, + nextIndex: leaf.nextIndex, + index: low.nextIndex, + sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), + }; + } + + static toBigints(leaf: Leaf): LeafValue { + return { + key: leaf.key.toBigInt(), + value: leaf.value.toBigInt(), + nextKey: leaf.nextKey.toBigInt(), + index: leaf.index.toBigInt(), + nextIndex: leaf.nextIndex.toBigInt(), + }; + } +} + +type LeafValue = { + value: bigint; + + key: bigint; + nextKey: bigint; + + index: bigint; + nextIndex: bigint; +}; + +class OptionField extends Option(Field) {} +class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} + // helper /** From 5c0723b278392f4eeebc74b72f59b7f66e0a25bb Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 11:34:19 +0200 Subject: [PATCH 37/93] first iteration of a provable intf (but won't work) --- src/lib/provable/merkle-tree-indexed.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 01eb052c57..53dcb7440c 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -52,6 +52,16 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { readonly sortedLeaves: LeafValue[]; }>; + static provable = provableFromClass(IndexedMerkleMap, { + root: Field, + length: Field, + height: Number, + data: Unconstrained.provableWithEmpty({ + nodes: [] as (bigint | undefined)[][], + sortedLeaves: [] as LeafValue[], + }), + }); + /** * Creates a new, empty Indexed Merkle Map, given its height. */ From 6ab8801e2d4928a199f73f1e35231da3c668d0bb Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 11:40:08 +0200 Subject: [PATCH 38/93] remove type intf --- src/lib/provable/merkle-tree-indexed.ts | 50 +++++++++++++------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 53dcb7440c..d38bd4093c 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -12,28 +12,7 @@ import { provableFromClass } from './types/provable-derivers.js'; export { IndexedMerkleMap }; -type IndexedMerkleMapBase = { - root: Field; - - // (lower-level) method to insert a new leaf `(key, value)`. proves that `key` doesn't exist yet - insert(key: Field, value: Field): void; - - // (lower-level) method to update an existing leaf `(key, value)`. proves that the `key` exists. - update(key: Field, value: Field): void; - - // method that performs _either_ an insertion or update, depending on whether the key exists - set(key: Field, value: Field): void; - - // method to get a value from a key. returns an option to account for the key not existing - // note: this has to prove that the option's `isSome` is correct - get(key: Field): Option; // the optional `Field` here is the value - - // optional / nice-to-have: remove a key and its value from the tree; proves that the key is included. - // (implementation: leave a wasted leaf in place but skip it in the linked list encoding) - // remove(key: Field): void; -}; - -class IndexedMerkleMap implements IndexedMerkleMapBase { +class IndexedMerkleMap { // data defining the provable interface of a tree root: Field; length: Field; // length of the leaves array @@ -55,7 +34,6 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { static provable = provableFromClass(IndexedMerkleMap, { root: Field, length: Field, - height: Number, data: Unconstrained.provableWithEmpty({ nodes: [] as (bigint | undefined)[][], sortedLeaves: [] as LeafValue[], @@ -96,6 +74,11 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.data = Unconstrained.from({ nodes, sortedLeaves: [firstLeaf] }); } + /** + * Insert a new leaf `(key, value)`. + * + * Proves that `key` doesn't exist yet. + */ insert(key: Field | bigint, value: Field | bigint) { key = Field(key); value = Field(value); @@ -127,6 +110,11 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(false, leaf); } + /** + * Update an existing leaf `(key, value)`. + * + * Proves that the `key` exists. + */ update(key: Field | bigint, value: Field | bigint) { key = Field(key); value = Field(value); @@ -142,6 +130,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(true, newSelf); } + /** + * Perform _either_ an insertion or update, depending on whether the key exists. + */ set(key: Field | bigint, value: Field | bigint) { key = Field(key); value = Field(value); @@ -179,6 +170,11 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { this.setLeafUnconstrained(keyExists, newLeaf); } + /** + * Get a value from a key. + * + * Returns an option which is `None` if the key doesn't exist. (In that case, the option's value is unconstrained.) + */ get(key: Field | bigint): Option { key = Field(key); @@ -200,6 +196,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { // helper methods + /** + * Helper method to prove inclusion of a leaf in the tree. + */ proveInclusion(leaf: Leaf, message?: string) { // TODO: here, we don't actually care about the index, // so we could add a mode where `computeRoot()` doesn't prove it @@ -208,6 +207,9 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); } + /** + * Helper method to conditionally prove inclusion of a leaf in the tree. + */ proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { let node = Leaf.hashNode(leaf); let root = this.computeRoot(leaf.index, node); @@ -218,7 +220,7 @@ class IndexedMerkleMap implements IndexedMerkleMapBase { } /** - * Compute the root given a leaf node and its index. + * Helper method to compute the root given a leaf node and its index. */ computeRoot(index: Field, node: Field) { let indexU = Unconstrained.witness(() => Number(index.toBigInt())); From d123aa94be78832c35b99a1e2234937722a60af2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 12:14:44 +0200 Subject: [PATCH 39/93] make it a class factory --- src/lib/provable/merkle-tree-indexed.ts | 85 +++++++++++++------ .../provable/test/merkle-tree.unit-test.ts | 5 +- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index d38bd4093c..1e58741891 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -2,7 +2,7 @@ import { Poseidon as PoseidonBigint } from '../../bindings/crypto/poseidon.js'; import { Bool, Field } from './wrapped.js'; import { Option } from './option.js'; import { Struct } from './types/struct.js'; -import { InferValue } from 'src/bindings/lib/provable-generic.js'; +import { InferValue } from '../../bindings/lib/provable-generic.js'; import { assert } from './gadgets/common.js'; import { Unconstrained } from './types/unconstrained.js'; import { Provable } from './provable.js'; @@ -12,11 +12,49 @@ import { provableFromClass } from './types/provable-derivers.js'; export { IndexedMerkleMap }; -class IndexedMerkleMap { +/** + * Class factory for an Indexed Merkle Map with a given height. + * + * ```ts + * let map = new (IndexedMerkleMap(3))(); + * + * map.set(2n, 14n); + * map.set(1n, 13n); + * + * let x = map.get(2n).assertSome(); // 14 + * ``` + */ +function IndexedMerkleMap(height: number) { + return class IndexedMerkleMap extends IndexedMerkleMapAbstract { + constructor() { + // we can't access the abstract `height` property in the base constructor + super(height); + } + + get height() { + return height; + } + + static provable = provableFromClass(IndexedMerkleMap, provableBase); + }; +} + +const provableBase = { + root: Field, + length: Field, + data: Unconstrained.provableWithEmpty({ + nodes: [] as (bigint | undefined)[][], + sortedLeaves: [] as LeafValue[], + }), +}; + +abstract class IndexedMerkleMapAbstract { // data defining the provable interface of a tree root: Field; length: Field; // length of the leaves array - readonly height: number; + + // static data defining constraints + abstract get height(): number; // the raw data stored in the tree, plus helper structures readonly data: Unconstrained<{ @@ -31,14 +69,11 @@ class IndexedMerkleMap { readonly sortedLeaves: LeafValue[]; }>; - static provable = provableFromClass(IndexedMerkleMap, { - root: Field, - length: Field, - data: Unconstrained.provableWithEmpty({ - nodes: [] as (bigint | undefined)[][], - sortedLeaves: [] as LeafValue[], - }), - }); + // we'd like to do `abstract static provable` here but that's not supported + static provable: Provable< + IndexedMerkleMapAbstract, + InferValue + > = undefined as any; /** * Creates a new, empty Indexed Merkle Map, given its height. @@ -49,7 +84,6 @@ class IndexedMerkleMap { height < 53, 'height must be less than 53, so that we can use 64-bit floats to represent indices.' ); - this.height = height; let nodes: (bigint | undefined)[][] = Array(height); for (let level = 0; level < height; level++) { @@ -84,7 +118,7 @@ class IndexedMerkleMap { value = Field(value); // prove that the key doesn't exist yet by presenting a valid low node - let low = Provable.witness(Leaf, () => this.findLeaf(key).low); + let low = Provable.witness(Leaf, () => this._findLeaf(key).low); this.proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); @@ -95,7 +129,7 @@ class IndexedMerkleMap { let index = this.length; let newLow = { ...low, nextKey: key, nextIndex: index }; this.root = this.computeRoot(newLow.index, Leaf.hashNode(newLow)); - this.setLeafUnconstrained(true, newLow); + this._setLeafUnconstrained(true, newLow); // create and append new leaf let leaf = Leaf.nextAfter(newLow, { @@ -107,7 +141,7 @@ class IndexedMerkleMap { this.root = this.computeRoot(index, Leaf.hashNode(leaf)); this.length = this.length.add(1); - this.setLeafUnconstrained(false, leaf); + this._setLeafUnconstrained(false, leaf); } /** @@ -120,14 +154,14 @@ class IndexedMerkleMap { value = Field(value); // prove that the key exists by presenting a leaf that contains it - let self = Provable.witness(Leaf, () => this.findLeaf(key).self); + let self = Provable.witness(Leaf, () => this._findLeaf(key).self); this.proveInclusion(self, 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf (key)'); // update leaf let newSelf = { ...self, value }; this.root = this.computeRoot(self.index, Leaf.hashNode(newSelf)); - this.setLeafUnconstrained(true, newSelf); + this._setLeafUnconstrained(true, newSelf); } /** @@ -138,7 +172,7 @@ class IndexedMerkleMap { value = Field(value); // prove whether the key exists or not, by showing a valid low node - let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); + let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); this.proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); @@ -156,7 +190,7 @@ class IndexedMerkleMap { // update low node, or leave it as is let newLow = { ...low, nextKey: key, nextIndex: index }; this.root = this.computeRoot(low.index, Leaf.hashNode(newLow)); - this.setLeafUnconstrained(true, newLow); + this._setLeafUnconstrained(true, newLow); // update leaf, or append a new one let newLeaf = Leaf.nextAfter(newLow, { @@ -167,7 +201,7 @@ class IndexedMerkleMap { }); this.root = this.computeRoot(index, Leaf.hashNode(newLeaf)); this.length = Provable.if(keyExists, this.length, this.length.add(1)); - this.setLeafUnconstrained(keyExists, newLeaf); + this._setLeafUnconstrained(keyExists, newLeaf); } /** @@ -179,7 +213,7 @@ class IndexedMerkleMap { key = Field(key); // prove whether the key exists or not, by showing a valid low node - let { low, self } = Provable.witness(LeafPair, () => this.findLeaf(key)); + let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); this.proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); @@ -254,9 +288,9 @@ class IndexedMerkleMap { * * If the key does not exist, a dummy value is returned for the leaf. * - * Assumes to run outside provable code. + * Can only be called outside provable code. */ - findLeaf(key_: Field | bigint): InferValue { + _findLeaf(key_: Field | bigint): InferValue { let key = typeof key_ === 'bigint' ? key_ : key_.toBigInt(); assert(key >= 0n, 'key must be positive'); let leaves = this.data.get().sortedLeaves; @@ -286,7 +320,7 @@ class IndexedMerkleMap { /** * Update or append a leaf in our internal data structures */ - private setLeafUnconstrained(leafExists: Bool | boolean, leaf: Leaf) { + _setLeafUnconstrained(leafExists: Bool | boolean, leaf: Leaf) { Provable.asProver(() => { let { nodes, sortedLeaves } = this.data.get(); @@ -414,9 +448,10 @@ type LeafValue = { nextIndex: bigint; }; -class OptionField extends Option(Field) {} class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} +class OptionField extends Option(Field) {} + // helper /** diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 5c8778d546..721d57a311 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -6,19 +6,16 @@ import { MerkleMap } from '../merkle-map.js'; import { IndexedMerkleMap } from '../merkle-tree-indexed.js'; console.log('new map'); -let map = new IndexedMerkleMap(3); +let map = new (IndexedMerkleMap(3))(); console.log('insert 2 and 1'); map.insert(2n, 14n); map.insert(1n, 13n); // console.dir(map.findLeaf(2n), { depth: null }); -console.log('test'); expect(map.get(1n).assertSome().toBigInt()).toEqual(13n); expect(map.get(2n).assertSome().toBigInt()).toEqual(14n); -console.dir(map.data.get().sortedLeaves, { depth: null }); - console.log('update 2 and 0'); map.update(2n, 15n); map.update(0n, 12n); From 4e0b1743ff7de203b9c724b2c7b9c710dfad112c Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 12:47:49 +0200 Subject: [PATCH 40/93] some refinements and fixes --- src/lib/provable/merkle-tree-indexed.ts | 64 ++++++++++++------- .../provable/test/merkle-tree.unit-test.ts | 50 ++++++++++----- 2 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 1e58741891..bc7dc89c98 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -2,7 +2,7 @@ import { Poseidon as PoseidonBigint } from '../../bindings/crypto/poseidon.js'; import { Bool, Field } from './wrapped.js'; import { Option } from './option.js'; import { Struct } from './types/struct.js'; -import { InferValue } from '../../bindings/lib/provable-generic.js'; +import { From, InferValue } from '../../bindings/lib/provable-generic.js'; import { assert } from './gadgets/common.js'; import { Unconstrained } from './types/unconstrained.js'; import { Provable } from './provable.js'; @@ -10,8 +10,12 @@ import { Poseidon } from './crypto/poseidon.js'; import { conditionalSwap } from './merkle-tree.js'; import { provableFromClass } from './types/provable-derivers.js'; +// external API export { IndexedMerkleMap }; +// internal API +export { Leaf }; + /** * Class factory for an Indexed Merkle Map with a given height. * @@ -98,9 +102,7 @@ abstract class IndexedMerkleMapAbstract { index: 0n, nextIndex: 0n, // TODO: ok? }; - let firstNode = Leaf.hashNode( - Leaf.fromValue({ ...firstLeaf, sortedIndex: Unconstrained.from(0) }) - ); + let firstNode = Leaf.hashNode(firstLeaf); let root = Nodes.setLeafNode(nodes, 0, firstNode.toBigInt()); this.root = Field(root); this.length = Field(1); @@ -117,6 +119,10 @@ abstract class IndexedMerkleMapAbstract { key = Field(key); value = Field(value); + // check that we can insert a new leaf, by asserting the length fits in the tree + let index = this.length; + let indexBits = index.toBits(this.height - 1); + // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); this.proveInclusion(low, 'Invalid low node (root)'); @@ -126,7 +132,6 @@ abstract class IndexedMerkleMapAbstract { key.assertLessThan(low.nextKey, 'Key already exists in the tree'); // update low node - let index = this.length; let newLow = { ...low, nextKey: key, nextIndex: index }; this.root = this.computeRoot(newLow.index, Leaf.hashNode(newLow)); this._setLeafUnconstrained(true, newLow); @@ -139,7 +144,7 @@ abstract class IndexedMerkleMapAbstract { nextIndex: low.nextIndex, }); - this.root = this.computeRoot(index, Leaf.hashNode(leaf)); + this.root = this.computeRoot(indexBits, Leaf.hashNode(leaf)); this.length = this.length.add(1); this._setLeafUnconstrained(false, leaf); } @@ -255,10 +260,18 @@ abstract class IndexedMerkleMapAbstract { /** * Helper method to compute the root given a leaf node and its index. + * + * The index can be given as a `Field` or as an array of bits. */ - computeRoot(index: Field, node: Field) { - let indexU = Unconstrained.witness(() => Number(index.toBigInt())); - let indexBits = index.toBits(this.height - 1); + computeRoot(index: Field | Bool[], node: Field) { + let indexBits = + index instanceof Field ? index.toBits(this.height - 1) : index; + + assert(indexBits.length === this.height - 1, `Invalid index size`); + + let indexU = Unconstrained.witness(() => + Number(Field.fromBits(indexBits).toBigInt()) + ); for (let level = 0; level < this.height - 1; level++) { // in every iteration, we witness a sibling and hash it to get the parent node @@ -268,14 +281,9 @@ abstract class IndexedMerkleMapAbstract { indexU.set(i >> 1); let isLeft = !isRight.toBoolean(); let nodes = this.data.get().nodes; - let sibling = Nodes.getNode( - nodes, - level, - isLeft ? i + 1 : i - 1, - false - ); - return sibling; + return Nodes.getNode(nodes, level, isLeft ? i + 1 : i - 1, false); }); + let [right, left] = conditionalSwap(isRight, node, sibling); node = Poseidon.hash([left, right]); } @@ -394,12 +402,12 @@ namespace Nodes { // leaf -type BaseLeaf = { - key: Field; - value: Field; - nextKey: Field; - nextIndex: Field; -}; +class BaseLeaf extends Struct({ + key: Field, + value: Field, + nextKey: Field, + nextIndex: Field, +}) {} class Leaf extends Struct({ value: Field, @@ -412,10 +420,18 @@ class Leaf extends Struct({ sortedIndex: Unconstrained.provableWithEmpty(0), }) { - static hashNode({ key, value, nextKey, nextIndex }: BaseLeaf) { - return Poseidon.hash([key, value, nextKey, nextIndex]); + /** + * Compute a leaf node: the hash of a leaf that becomes part of the Merkle tree. + */ + static hashNode(leaf: From) { + // note: we don't have to include the `index` in the leaf hash, + // because computing the root already commits to the index + return Poseidon.hashPacked(BaseLeaf, BaseLeaf.fromValue(leaf)); } + /** + * Create a new leaf, given its low node. + */ static nextAfter(low: Leaf, leaf: BaseLeaf): Leaf { return { key: leaf.key, diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 721d57a311..e76f1ec8af 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -3,27 +3,45 @@ import { conditionalSwap } from '../merkle-tree.js'; import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; import { MerkleMap } from '../merkle-map.js'; -import { IndexedMerkleMap } from '../merkle-tree-indexed.js'; +import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; -console.log('new map'); -let map = new (IndexedMerkleMap(3))(); +// some manual tests for IndexedMerkleMap +{ + let map = new (IndexedMerkleMap(3))(); -console.log('insert 2 and 1'); -map.insert(2n, 14n); -map.insert(1n, 13n); -// console.dir(map.findLeaf(2n), { depth: null }); + map.insert(2n, 14n); + map.insert(1n, 13n); -expect(map.get(1n).assertSome().toBigInt()).toEqual(13n); -expect(map.get(2n).assertSome().toBigInt()).toEqual(14n); + expect(map.get(1n).assertSome().toBigInt()).toEqual(13n); + expect(map.get(2n).assertSome().toBigInt()).toEqual(14n); + expect(map.get(3n).isSome.toBoolean()).toEqual(false); -console.log('update 2 and 0'); -map.update(2n, 15n); -map.update(0n, 12n); -// TODO get() doesn't work on 0n because the low node checks fail -// expect(map.get(0n).assertSome().toBigInt()).toEqual(12n); -expect(map.get(2n).assertSome().toBigInt()).toEqual(15n); + map.update(2n, 15n); + map.update(0n, 12n); + expect(map.get(2n).assertSome().toBigInt()).toEqual(15n); -console.dir(map.data.get().sortedLeaves, { depth: null }); + // TODO get() doesn't work on 0n because the low node checks fail + // expect(map.get(0n).assertSome().toBigInt()).toEqual(12n); + + // can't insert the same key twice + expect(() => map.insert(1n, 17n)).toThrow('Key already exists'); + + map.set(4n, 16n); + map.set(1n, 17n); + expect(map.get(4n).assertSome().toBigInt()).toEqual(16n); + expect(map.get(1n).assertSome().toBigInt()).toEqual(17n); + expect(map.get(5n).isSome.toBoolean()).toEqual(false); + + // can't insert more than 2^(height - 1) = 2^2 = 4 keys + expect(() => map.insert(8n, 19n)).toThrow('4 does not fit in 2 bits'); + + // check nodes against `MerkleTree` implementation + let keys = [0n, 2n, 1n, 4n]; // insertion order + let leaves = keys.map((key) => Leaf.hashNode(map._findLeaf(key).self)); + + console.dir(map.data.get().sortedLeaves, { depth: null }); + console.dir(map.data.get().nodes, { depth: null }); +} test(Random.bool, Random.field, Random.field, (b, x, y) => { let [x0, y0] = conditionalSwap(Bool(!!b), Field(x), Field(y)); From a84eb65f995b62568725f3199b1d165c49316c81 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 12:55:14 +0200 Subject: [PATCH 41/93] test against MerkleTree --- src/lib/provable/test/merkle-tree.unit-test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index e76f1ec8af..69bc6b2461 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -1,5 +1,5 @@ import { Bool, Field } from '../wrapped.js'; -import { conditionalSwap } from '../merkle-tree.js'; +import { MerkleTree, conditionalSwap } from '../merkle-tree.js'; import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; import { MerkleMap } from '../merkle-map.js'; @@ -35,12 +35,20 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; // can't insert more than 2^(height - 1) = 2^2 = 4 keys expect(() => map.insert(8n, 19n)).toThrow('4 does not fit in 2 bits'); - // check nodes against `MerkleTree` implementation + // check that internal nodes exactly match `MerkleTree` implementation let keys = [0n, 2n, 1n, 4n]; // insertion order let leaves = keys.map((key) => Leaf.hashNode(map._findLeaf(key).self)); + let tree = new MerkleTree(3); + tree.fill(leaves); + let nodes = map.data.get().nodes; + + for (let level = 0; level < 3; level++) { + for (let i = 0; i < 2 ** (2 - level); i++) { + expect(nodes[level][i]).toEqual(tree.nodes[level][i].toBigInt()); + } + } console.dir(map.data.get().sortedLeaves, { depth: null }); - console.dir(map.data.get().nodes, { depth: null }); } test(Random.bool, Random.field, Random.field, (b, x, y) => { From 4fede55e3de67132f7c267b7f2fd820c25b4ed72 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 13:16:48 +0200 Subject: [PATCH 42/93] assert validity before all mutations --- src/lib/provable/merkle-tree-indexed.ts | 13 ++++++++++--- src/lib/provable/test/merkle-tree.unit-test.ts | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index bc7dc89c98..b3e2bba8f4 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -131,6 +131,8 @@ abstract class IndexedMerkleMapAbstract { // if the key does exist, we have lowNode.nextKey == key, and this line fails key.assertLessThan(low.nextKey, 'Key already exists in the tree'); + // at this point, we know that we have a valid insertion; so we can mutate internal data + // update low node let newLow = { ...low, nextKey: key, nextIndex: index }; this.root = this.computeRoot(newLow.index, Leaf.hashNode(newLow)); @@ -163,6 +165,8 @@ abstract class IndexedMerkleMapAbstract { this.proveInclusion(self, 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf (key)'); + // at this point, we know that we have a valid update; so we can mutate internal data + // update leaf let newSelf = { ...self, value }; this.root = this.computeRoot(self.index, Leaf.hashNode(newSelf)); @@ -185,12 +189,15 @@ abstract class IndexedMerkleMapAbstract { // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); + // the leaf's index depends on whether it exists + let index = Provable.if(keyExists, low.nextIndex, this.length); + let indexBits = index.toBits(this.height - 1); + // prove inclusion of this leaf if it exists this.proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); - // the leaf's index depends on whether it exists - let index = Provable.if(keyExists, low.nextIndex, this.length); + // at this point, we know that we have a valid update or insertion; so we can mutate internal data // update low node, or leave it as is let newLow = { ...low, nextKey: key, nextIndex: index }; @@ -204,7 +211,7 @@ abstract class IndexedMerkleMapAbstract { nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), nextIndex: Provable.if(keyExists, self.nextIndex, low.nextIndex), }); - this.root = this.computeRoot(index, Leaf.hashNode(newLeaf)); + this.root = this.computeRoot(indexBits, Leaf.hashNode(newLeaf)); this.length = Provable.if(keyExists, this.length, this.length.add(1)); this._setLeafUnconstrained(keyExists, newLeaf); } diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 69bc6b2461..0948a81eee 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -12,6 +12,11 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; map.insert(2n, 14n); map.insert(1n, 13n); + // -1 (the max value) can't be inserted, because there's always a value pointing to it, + // and yet it's not included as a leaf + expect(() => map.insert(-1n, 11n)).toThrow('Key already exists'); + expect(() => map.set(-1n, 12n)).toThrow('Invalid leaf'); + expect(map.get(1n).assertSome().toBigInt()).toEqual(13n); expect(map.get(2n).assertSome().toBigInt()).toEqual(14n); expect(map.get(3n).isSome.toBoolean()).toEqual(false); @@ -34,6 +39,7 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; // can't insert more than 2^(height - 1) = 2^2 = 4 keys expect(() => map.insert(8n, 19n)).toThrow('4 does not fit in 2 bits'); + expect(() => map.set(8n, 19n)).toThrow('4 does not fit in 2 bits'); // check that internal nodes exactly match `MerkleTree` implementation let keys = [0n, 2n, 1n, 4n]; // insertion order From dda00298f5587e28c2643619477199af056b647e Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 13:25:41 +0200 Subject: [PATCH 43/93] another manual test --- .../provable/test/merkle-tree.unit-test.ts | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 0948a81eee..5ffc358562 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -43,9 +43,9 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; // check that internal nodes exactly match `MerkleTree` implementation let keys = [0n, 2n, 1n, 4n]; // insertion order - let leaves = keys.map((key) => Leaf.hashNode(map._findLeaf(key).self)); + let leafNodes = keys.map((key) => Leaf.hashNode(map._findLeaf(key).self)); let tree = new MerkleTree(3); - tree.fill(leaves); + tree.fill(leafNodes); let nodes = map.data.get().nodes; for (let level = 0; level < 3; level++) { @@ -54,7 +54,26 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; } } - console.dir(map.data.get().sortedLeaves, { depth: null }); + // check that internal `sortedLeaves` are as expected + + // data sorted by key: + let sorted = [ + { key: 0n, value: 12n, index: 0n }, + { key: 1n, value: 17n, index: 2n }, + { key: 2n, value: 15n, index: 1n }, + { key: 4n, value: 16n, index: 3n }, + ]; + let sortedLeaves = map.data.get().sortedLeaves; + + for (let i = 0; i < 4; i++) { + expect(sortedLeaves[i]).toEqual({ + key: sorted[i].key, + value: sorted[i].value, + nextKey: sorted[i + 1]?.key ?? Field.ORDER - 1n, + index: sorted[i].index, + nextIndex: sorted[i + 1]?.index ?? 0n, + }); + } } test(Random.bool, Random.field, Random.field, (b, x, y) => { From fd8f20c2e1fddd0fd3fd6e9d174bcab6ca82f682 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 14:29:21 +0200 Subject: [PATCH 44/93] expose first leaf --- src/lib/provable/merkle-tree-indexed.ts | 19 ++++++++++--------- .../provable/test/merkle-tree.unit-test.ts | 13 +++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index b3e2bba8f4..fb0344d534 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -94,21 +94,22 @@ abstract class IndexedMerkleMapAbstract { nodes[level] = []; } - let firstLeaf = { - key: 0n, - value: 0n, - // maximum, which is always greater than any key that is a hash - nextKey: Field.ORDER - 1n, - index: 0n, - nextIndex: 0n, // TODO: ok? - }; - let firstNode = Leaf.hashNode(firstLeaf); + let firstLeaf = IndexedMerkleMapAbstract._firstLeaf; + let firstNode = Leaf.hashNode(IndexedMerkleMapAbstract._firstLeaf); let root = Nodes.setLeafNode(nodes, 0, firstNode.toBigInt()); this.root = Field(root); this.length = Field(1); this.data = Unconstrained.from({ nodes, sortedLeaves: [firstLeaf] }); } + static _firstLeaf = { + key: 0n, + value: 0n, + // maximum, which is always greater than any key that is a hash + nextKey: Field.ORDER - 1n, + index: 0n, + nextIndex: 0n, // TODO: ok? + }; /** * Insert a new leaf `(key, value)`. diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 5ffc358562..3c3d1c8247 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -9,6 +9,13 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; { let map = new (IndexedMerkleMap(3))(); + // there's 1 element in the map at the beginning + // check initial root against `MerkleTree` implementation + expect(map.length.toBigInt()).toEqual(1n); + let initialTree = new MerkleTree(3); + initialTree.setLeaf(0n, Leaf.hashNode(IndexedMerkleMap(3)._firstLeaf)); + expect(map.root).toEqual(initialTree.getRoot()); + map.insert(2n, 14n); map.insert(1n, 13n); @@ -31,6 +38,9 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; // can't insert the same key twice expect(() => map.insert(1n, 17n)).toThrow('Key already exists'); + // can't update a non-existent key + expect(() => map.update(3n, 16n)).toThrow('Key does not exist'); + map.set(4n, 16n); map.set(1n, 17n); expect(map.get(4n).assertSome().toBigInt()).toEqual(16n); @@ -41,6 +51,9 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; expect(() => map.insert(8n, 19n)).toThrow('4 does not fit in 2 bits'); expect(() => map.set(8n, 19n)).toThrow('4 does not fit in 2 bits'); + // check that length is as expected + expect(map.length.toBigInt()).toEqual(4n); + // check that internal nodes exactly match `MerkleTree` implementation let keys = [0n, 2n, 1n, 4n]; // insertion order let leafNodes = keys.map((key) => Leaf.hashNode(map._findLeaf(key).self)); From f173c596bdfdd97e45eb7a897bc672d34ff3c881 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 15:24:22 +0200 Subject: [PATCH 45/93] provable tests --- .../provable/test/merkle-tree.unit-test.ts | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 3c3d1c8247..4206c16769 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -4,6 +4,8 @@ import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; import { MerkleMap } from '../merkle-map.js'; import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; +import { synchronousRunners } from '../core/provable-context.js'; +import { Provable } from '../provable.js'; // some manual tests for IndexedMerkleMap { @@ -89,6 +91,93 @@ import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; } } +// property tests for indexed merkle map + +let uniformField = Random(Field.random); +let { runAndCheckSync } = await synchronousRunners(); + +let n = 5; + +test( + Random.array(uniformField, n), + Random.array(uniformField, n), + Random.array(Random.field, n), + Random.array(Random.field, n), + Random.int(6, 40), + Random.int(0, n - 1), + (initialKeys, keys, initialValues, values, height, i0) => { + class MerkleMap extends IndexedMerkleMap(height) {} + + // fill a merkle map with the initial keys and values outside provable code + let map = new MerkleMap(); + for (let i = 0; i < n; i++) { + map.insert(initialKeys[i], initialValues[i]); + } + + // pass the map to a circuit + runAndCheckSync(() => { + map = Provable.witness(MerkleMap.provable, () => map); + + for (let i = 0; i < n; i++) { + // confirm we still have the same keys and values + map.get(initialKeys[i]).assertSome().assertEquals(initialValues[i]); + + // new keys are not in the map + map.get(keys[i]).isSome.assertFalse(); + } + + // can't update a non-existent key + expect(() => map.update(keys[i0], values[i0])).toThrow( + 'Key does not exist' + ); + + // set initial values at new keys, and values at initial keys + for (let i = 0; i < n; i++) { + map.set(keys[i], initialValues[i]); + map.set(initialKeys[i], values[i]); + } + + // check that the updated keys and values are in the map + for (let i = 0; i < n; i++) { + map.get(keys[i]).assertSome().assertEquals(initialValues[i]); + map.get(initialKeys[i]).assertSome().assertEquals(values[i]); + } + + // update the new keys with the new values + for (let i = 0; i < n; i++) { + map.update(keys[i], values[i]); + } + + // move the map back to constants + Provable.asProver(() => { + map = Provable.toConstant(MerkleMap.provable, map); + }); + }); + + // check that the map is still the same + for (let i = 0; i < n; i++) { + expect(map.get(keys[i]).assertSome()).toEqual(Field(values[i])); + expect(map.get(initialKeys[i]).assertSome()).toEqual(Field(values[i])); + } + // random element is not in the map + expect(map.get(Field.random()).isSome).toEqual(Bool(false)); + // length is as expected + expect(map.length).toEqual(Field(2 * n + 1)); + + // creating a new map with the same key-value pairs, where keys are inserted in the same order, gives the same root + let map2 = new MerkleMap(); + for (let i = 0; i < n; i++) { + map2.insert(initialKeys[i], values[i]); + } + for (let i = 0; i < n; i++) { + map2.insert(keys[i], values[i]); + } + expect(map.root).toEqual(map2.root); + } +); + +// property tests for conditional swap + test(Random.bool, Random.field, Random.field, (b, x, y) => { let [x0, y0] = conditionalSwap(Bool(!!b), Field(x), Field(y)); @@ -102,6 +191,8 @@ test(Random.bool, Random.field, Random.field, (b, x, y) => { } }); +// property tests for merkle map + test(Random.field, (key) => { let map = new MerkleMap(); let witness = map.getWitness(Field(key)); From 58304537e3173c5371145aab7fe2433dced240b7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 15:36:40 +0200 Subject: [PATCH 46/93] more documentation --- src/lib/provable/merkle-tree-indexed.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index fb0344d534..6577be61c5 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -20,13 +20,34 @@ export { Leaf }; * Class factory for an Indexed Merkle Map with a given height. * * ```ts - * let map = new (IndexedMerkleMap(3))(); + * class MerkleMap extends IndexedMerkleMap(3) {} + * + * let map = new MerkleMap(); * * map.set(2n, 14n); * map.set(1n, 13n); * * let x = map.get(2n).assertSome(); // 14 * ``` + * + * Indexed Merkle maps can be used directly in provable code: + * + * ```ts + * ZkProgram({ + * methods: { + * test: { + * privateInputs: [MerkleMap.provable, Field], + * method(map: MerkleMap, key: Field) { + * // get the value associated with `key` + * let value = map.get(key).orElse(0n); + * + * // increment the value by 1 + * map.set(key, value.add(1)); + * } + * } + * } + * }) + * ``` */ function IndexedMerkleMap(height: number) { return class IndexedMerkleMap extends IndexedMerkleMapAbstract { @@ -222,7 +243,7 @@ abstract class IndexedMerkleMapAbstract { * * Returns an option which is `None` if the key doesn't exist. (In that case, the option's value is unconstrained.) */ - get(key: Field | bigint): Option { + get(key: Field | bigint): Option { key = Field(key); // prove whether the key exists or not, by showing a valid low node From 9daa83e0d320f9311feb8e5eab41e663f3907fc6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 16:12:35 +0200 Subject: [PATCH 47/93] some testing enhancements --- src/lib/provable/core/provable-context.ts | 21 ++++++++++++++------- src/lib/testing/constraint-system.ts | 6 ++++++ src/lib/testing/equivalent.ts | 7 ++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/lib/provable/core/provable-context.ts b/src/lib/provable/core/provable-context.ts index 29b258c02b..a697c5d00d 100644 --- a/src/lib/provable/core/provable-context.ts +++ b/src/lib/provable/core/provable-context.ts @@ -26,6 +26,7 @@ export { inCompileMode, gatesFromJson, printGates, + summarizeGates, MlConstraintSystem, }; @@ -167,13 +168,7 @@ function constraintSystemToJS(cs: MlConstraintSystem) { printGates(gates); }, summary() { - let gateTypes: Partial> = {}; - gateTypes['Total rows'] = rows; - for (let gate of gates) { - gateTypes[gate.type] ??= 0; - gateTypes[gate.type]!++; - } - return gateTypes; + return summarizeGates(gates); }, }; } @@ -188,6 +183,18 @@ function gatesFromJson(cs: { gates: JsonGate[]; public_input_size: number }) { return { publicInputSize: cs.public_input_size, gates }; } +// collect a summary of the constraint system + +function summarizeGates(gates: Gate[]) { + let gateTypes: Partial> = {}; + gateTypes['Total rows'] = gates.length; + for (let gate of gates) { + gateTypes[gate.type] ??= 0; + gateTypes[gate.type]!++; + } + return gateTypes; +} + // print a constraint system function printGates(gates: Gate[]) { diff --git a/src/lib/testing/constraint-system.ts b/src/lib/testing/constraint-system.ts index 3b625820f7..13a7d952c5 100644 --- a/src/lib/testing/constraint-system.ts +++ b/src/lib/testing/constraint-system.ts @@ -15,6 +15,7 @@ import { test } from './property.js'; import { Undefined, ZkProgram } from '../proof-system/zkprogram.js'; import { printGates, + summarizeGates, synchronousRunners, } from '../provable/core/provable-context.js'; @@ -351,6 +352,11 @@ constraintSystem.size = map((gates) => gates.length); */ constraintSystem.print = map(printGates); +/** + * Get constraint system summary. + */ +constraintSystem.summary = map(summarizeGates); + function repeat( n: number, gates: GateType | readonly GateType[] diff --git a/src/lib/testing/equivalent.ts b/src/lib/testing/equivalent.ts index 58eeefc9fb..c4a713c90b 100644 --- a/src/lib/testing/equivalent.ts +++ b/src/lib/testing/equivalent.ts @@ -282,6 +282,11 @@ function spec(spec: { back: (x: S) => T; assertEqual?: (x: T, y: T, message: string) => void; }): Spec; +function spec(spec: { + rng: Random; + provable: Provable; + assertEqual?: (x: T, y: T, message: string) => void; +}): ProvableSpec; function spec(spec: { rng: Random; assertEqual?: (x: T, y: T, message: string) => void; @@ -327,7 +332,7 @@ let bool: ProvableSpec = { }; let boolean: Spec = fromRandom(Random.boolean); -function fieldWithRng(rng: Random): Spec { +function fieldWithRng(rng: Random): ProvableSpec { return { ...field, rng }; } From c1661e821941bf6ca1ed4f307067778b52f6a794 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 16:13:41 +0200 Subject: [PATCH 48/93] add constraint comparison --- .../provable/test/merkle-tree.unit-test.ts | 92 +++++++++++++++++-- 1 file changed, 82 insertions(+), 10 deletions(-) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 4206c16769..d9a3845b8c 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -2,10 +2,77 @@ import { Bool, Field } from '../wrapped.js'; import { MerkleTree, conditionalSwap } from '../merkle-tree.js'; import { Random, test } from '../../testing/property.js'; import { expect } from 'expect'; -import { MerkleMap } from '../merkle-map.js'; +import { MerkleMap, MerkleMapWitness } from '../merkle-map.js'; import { IndexedMerkleMap, Leaf } from '../merkle-tree-indexed.js'; import { synchronousRunners } from '../core/provable-context.js'; import { Provable } from '../provable.js'; +import { constraintSystem } from '../../testing/constraint-system.js'; +import { field } from '../../testing/equivalent.js'; +import { throwError } from './test-utils.js'; + +const height = 32; +const IndexedMap32 = IndexedMerkleMap(height); +const indexedMap = new IndexedMap32(); + +// compare constraints used by indexed merkle map vs sparse merkle map + +console.log( + 'indexed merkle map (get)', + constraintSystem.size({ from: [field] }, (key) => { + indexedMap.get(key); + }) +); + +console.log( + 'sparse merkle map (get)', + constraintSystem.size( + { from: [field, field, field, field] }, + (root, key, value) => { + let mapWitness = Provable.witness(MerkleMapWitness, () => + throwError('unused') + ); + let [actualRoot, actualKey] = mapWitness.computeRootAndKey(value); + key.assertEquals(actualKey); + root.assertEquals(actualRoot); + } + ) +); + +console.log( + '\nindexed merkle map (insert)', + constraintSystem.size({ from: [field, field] }, (key, value) => { + indexedMap.insert(key, value); + }) +); +console.log( + 'indexed merkle map (update)', + constraintSystem.size({ from: [field, field] }, (key, value) => { + indexedMap.update(key, value); + }) +); +console.log( + 'indexed merkle map (set)', + constraintSystem.size({ from: [field, field] }, (key, value) => { + indexedMap.set(key, value); + }) +); + +console.log( + 'sparse merkle map (set)', + constraintSystem.size( + { from: [field, field, field, field] }, + (root, key, oldValue, value) => { + let mapWitness = Provable.witness(MerkleMapWitness, () => + throwError('unused') + ); + let [actualRoot, actualKey] = mapWitness.computeRootAndKey(oldValue); + key.assertEquals(actualKey); + root.assertEquals(actualRoot); + + let [_newRoot] = mapWitness.computeRootAndKey(value); + } + ) +); // some manual tests for IndexedMerkleMap { @@ -93,7 +160,7 @@ import { Provable } from '../provable.js'; // property tests for indexed merkle map -let uniformField = Random(Field.random); +let uniformField = Random.map(Random(Field.random), (x) => x.toBigInt()); let { runAndCheckSync } = await synchronousRunners(); let n = 5; @@ -114,38 +181,43 @@ test( map.insert(initialKeys[i], initialValues[i]); } + const witness = (x: Field | bigint) => Provable.witness(Field, () => x); + // pass the map to a circuit runAndCheckSync(() => { map = Provable.witness(MerkleMap.provable, () => map); + let initialKeysF = initialKeys.map(witness); + let keysF = keys.map(witness); + let valuesF = values.map(witness); for (let i = 0; i < n; i++) { // confirm we still have the same keys and values - map.get(initialKeys[i]).assertSome().assertEquals(initialValues[i]); + map.get(initialKeysF[i]).assertSome().assertEquals(initialValues[i]); // new keys are not in the map - map.get(keys[i]).isSome.assertFalse(); + map.get(keysF[i]).isSome.assertFalse(); } // can't update a non-existent key - expect(() => map.update(keys[i0], values[i0])).toThrow( + expect(() => map.update(keysF[i0], valuesF[i0])).toThrow( 'Key does not exist' ); // set initial values at new keys, and values at initial keys for (let i = 0; i < n; i++) { - map.set(keys[i], initialValues[i]); - map.set(initialKeys[i], values[i]); + map.set(keysF[i], initialValues[i]); + map.set(initialKeysF[i], valuesF[i]); } // check that the updated keys and values are in the map for (let i = 0; i < n; i++) { - map.get(keys[i]).assertSome().assertEquals(initialValues[i]); - map.get(initialKeys[i]).assertSome().assertEquals(values[i]); + map.get(keysF[i]).assertSome().assertEquals(initialValues[i]); + map.get(initialKeysF[i]).assertSome().assertEquals(valuesF[i]); } // update the new keys with the new values for (let i = 0; i < n; i++) { - map.update(keys[i], values[i]); + map.update(keys[i], valuesF[i]); } // move the map back to constants From 903bafe5000756d19dbb15e116ae1522e78e28ff Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 16:20:58 +0200 Subject: [PATCH 49/93] comment tweaks --- src/lib/provable/merkle-tree-indexed.ts | 3 ++- src/lib/provable/test/merkle-tree.unit-test.ts | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 6577be61c5..6f09e125ca 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -20,7 +20,7 @@ export { Leaf }; * Class factory for an Indexed Merkle Map with a given height. * * ```ts - * class MerkleMap extends IndexedMerkleMap(3) {} + * class MerkleMap extends IndexedMerkleMap(33) {} * * let map = new MerkleMap(); * @@ -37,6 +37,7 @@ export { Leaf }; * methods: { * test: { * privateInputs: [MerkleMap.provable, Field], + * * method(map: MerkleMap, key: Field) { * // get the value associated with `key` * let value = map.get(key).orElse(0n); diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index d9a3845b8c..66ba06e641 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -10,11 +10,11 @@ import { constraintSystem } from '../../testing/constraint-system.js'; import { field } from '../../testing/equivalent.js'; import { throwError } from './test-utils.js'; -const height = 32; -const IndexedMap32 = IndexedMerkleMap(height); -const indexedMap = new IndexedMap32(); +const height = 31; +const IndexedMap30 = IndexedMerkleMap(height); +const indexedMap = new IndexedMap30(); -// compare constraints used by indexed merkle map vs sparse merkle map +// compare constraints used by indexed merkle map (with 1B leaves) vs sparse merkle map console.log( 'indexed merkle map (get)', From f15254efa76e497bd224a4f4d9886e2df7d1ac5c Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 16:23:01 +0200 Subject: [PATCH 50/93] export --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index 0252a528bd..56216cd68a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,6 +52,7 @@ export { Gadgets } from './lib/provable/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; export { MerkleList, MerkleListIterator } from './lib/provable/merkle-list.js'; +export { IndexedMerkleMap } from './lib/provable/merkle-tree-indexed.js'; export { Option } from './lib/provable/option.js'; export * as Mina from './lib/mina/mina.js'; From 870b695cff034b952d3437633d671a24b2bb5c83 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 16:25:40 +0200 Subject: [PATCH 51/93] changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac3634036d..99f3eaf30f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased](https://github.com/o1-labs/o1js/compare/54d6545bf...HEAD) +### Added + +- `IndexedMerkleMap`, a better primitive for Merkleized storage which uses 4-8x fewer constraints than `MerkleMap` https://github.com/o1-labs/o1js/pull/1666 + - In contrast to `MerkleTree` and `MerkleMap`, `IndexedMerkleMap` has a high-level API that can be used in provable code. + ### Deprecated - `Int64.isPositive()` and `Int64.mod()` deprecated because they behave incorrectly on `-0` https://github.com/o1-labs/o1js/pull/1660 From cc5f09a2d74ef037db47fc730a15f29384456392 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 17:20:10 +0200 Subject: [PATCH 52/93] add non-optional get and make it the default --- src/lib/provable/merkle-tree-indexed.ts | 21 +++++++++- .../provable/test/merkle-tree.unit-test.ts | 39 ++++++++++++------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 6f09e125ca..38d48afbb9 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -239,12 +239,31 @@ abstract class IndexedMerkleMapAbstract { this._setLeafUnconstrained(keyExists, newLeaf); } + /** + * Get a value from a key. + * + * Proves that the key already exists in the map yet and fails otherwise. + */ + get(key: Field | bigint): Field { + key = Field(key); + + // prove that the key exists by presenting a leaf that contains it + let self = Provable.witness(Leaf, () => this._findLeaf(key).self); + this.proveInclusion(self, 'Key does not exist in the tree'); + self.key.assertEquals(key, 'Invalid leaf (key)'); + + return self.value; + } + /** * Get a value from a key. * * Returns an option which is `None` if the key doesn't exist. (In that case, the option's value is unconstrained.) + * + * Note that this is more flexible than `get()` and allows you to handle the case where the key doesn't exist. + * However, it uses about twice as many constraints for that reason. */ - get(key: Field | bigint): Option { + getOption(key: Field | bigint): Option { key = Field(key); // prove whether the key exists or not, by showing a valid low node diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 66ba06e641..6d2234223d 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -22,6 +22,12 @@ console.log( indexedMap.get(key); }) ); +console.log( + 'indexed merkle map (get option)', + constraintSystem.size({ from: [field] }, (key) => { + indexedMap.getOption(key); + }) +); console.log( 'sparse merkle map (get)', @@ -93,13 +99,13 @@ console.log( expect(() => map.insert(-1n, 11n)).toThrow('Key already exists'); expect(() => map.set(-1n, 12n)).toThrow('Invalid leaf'); - expect(map.get(1n).assertSome().toBigInt()).toEqual(13n); - expect(map.get(2n).assertSome().toBigInt()).toEqual(14n); - expect(map.get(3n).isSome.toBoolean()).toEqual(false); + expect(map.getOption(1n).assertSome().toBigInt()).toEqual(13n); + expect(map.getOption(2n).assertSome().toBigInt()).toEqual(14n); + expect(map.getOption(3n).isSome.toBoolean()).toEqual(false); map.update(2n, 15n); map.update(0n, 12n); - expect(map.get(2n).assertSome().toBigInt()).toEqual(15n); + expect(map.getOption(2n).assertSome().toBigInt()).toEqual(15n); // TODO get() doesn't work on 0n because the low node checks fail // expect(map.get(0n).assertSome().toBigInt()).toEqual(12n); @@ -112,9 +118,9 @@ console.log( map.set(4n, 16n); map.set(1n, 17n); - expect(map.get(4n).assertSome().toBigInt()).toEqual(16n); - expect(map.get(1n).assertSome().toBigInt()).toEqual(17n); - expect(map.get(5n).isSome.toBoolean()).toEqual(false); + expect(map.get(4n).toBigInt()).toEqual(16n); + expect(map.getOption(1n).assertSome().toBigInt()).toEqual(17n); + expect(map.getOption(5n).isSome.toBoolean()).toEqual(false); // can't insert more than 2^(height - 1) = 2^2 = 4 keys expect(() => map.insert(8n, 19n)).toThrow('4 does not fit in 2 bits'); @@ -192,10 +198,13 @@ test( for (let i = 0; i < n; i++) { // confirm we still have the same keys and values - map.get(initialKeysF[i]).assertSome().assertEquals(initialValues[i]); + map + .getOption(initialKeysF[i]) + .assertSome() + .assertEquals(initialValues[i]); // new keys are not in the map - map.get(keysF[i]).isSome.assertFalse(); + map.getOption(keysF[i]).isSome.assertFalse(); } // can't update a non-existent key @@ -211,8 +220,8 @@ test( // check that the updated keys and values are in the map for (let i = 0; i < n; i++) { - map.get(keysF[i]).assertSome().assertEquals(initialValues[i]); - map.get(initialKeysF[i]).assertSome().assertEquals(valuesF[i]); + map.getOption(keysF[i]).assertSome().assertEquals(initialValues[i]); + map.get(initialKeysF[i]).assertEquals(valuesF[i]); } // update the new keys with the new values @@ -228,11 +237,13 @@ test( // check that the map is still the same for (let i = 0; i < n; i++) { - expect(map.get(keys[i]).assertSome()).toEqual(Field(values[i])); - expect(map.get(initialKeys[i]).assertSome()).toEqual(Field(values[i])); + expect(map.getOption(keys[i]).assertSome()).toEqual(Field(values[i])); + expect(map.getOption(initialKeys[i]).assertSome()).toEqual( + Field(values[i]) + ); } // random element is not in the map - expect(map.get(Field.random()).isSome).toEqual(Bool(false)); + expect(map.getOption(Field.random()).isSome).toEqual(Bool(false)); // length is as expected expect(map.length).toEqual(Field(2 * n + 1)); From 9301b88170ba847c46ebb35ac8fe539c82090ac4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 17:25:20 +0200 Subject: [PATCH 53/93] move to experimental for now --- CHANGELOG.md | 2 +- src/index.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f3eaf30f..05666e171b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added -- `IndexedMerkleMap`, a better primitive for Merkleized storage which uses 4-8x fewer constraints than `MerkleMap` https://github.com/o1-labs/o1js/pull/1666 +- `Experimental.IndexedMerkleMap`, a better primitive for Merkleized storage which uses 4-8x fewer constraints than `MerkleMap` https://github.com/o1-labs/o1js/pull/1666 - In contrast to `MerkleTree` and `MerkleMap`, `IndexedMerkleMap` has a high-level API that can be used in provable code. ### Deprecated diff --git a/src/index.ts b/src/index.ts index 56216cd68a..296b3c95f2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,7 @@ export { Gadgets } from './lib/provable/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; export { MerkleList, MerkleListIterator } from './lib/provable/merkle-list.js'; -export { IndexedMerkleMap } from './lib/provable/merkle-tree-indexed.js'; +import { IndexedMerkleMap } from './lib/provable/merkle-tree-indexed.js'; export { Option } from './lib/provable/option.js'; export * as Mina from './lib/mina/mina.js'; @@ -134,6 +134,7 @@ export { Experimental }; const Experimental_ = { memoizeWitness, + IndexedMerkleMap, }; /** @@ -143,6 +144,9 @@ const Experimental_ = { namespace Experimental { export let memoizeWitness = Experimental_.memoizeWitness; + // indexed merkle map + export let IndexedMerkleMap = Experimental_.IndexedMerkleMap; + // offchain state export let OffchainState = OffchainState_.OffchainState; From cd973d706773ab0e33fb082bd4f34b26d2f07010 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 17:38:10 +0200 Subject: [PATCH 54/93] add methods that are only about inclusion of a key --- src/lib/provable/merkle-tree-indexed.ts | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 38d48afbb9..c9f09f7229 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -282,6 +282,51 @@ abstract class IndexedMerkleMapAbstract { return new OptionField({ isSome: keyExists, value: self.value }); } + // methods to check for inclusion for a key without being concerned about the value + + /** + * Prove that the given key exists in the map. + */ + assertIncluded(key: Field | bigint, message?: string) { + key = Field(key); + + // prove that the key exists by presenting a leaf that contains it + let self = Provable.witness(Leaf, () => this._findLeaf(key).self); + this.proveInclusion(self, message ?? 'Key does not exist in the tree'); + self.key.assertEquals(key, 'Invalid leaf (key)'); + } + + /** + * Prove that the given key does not exist in the map. + */ + assertNotIncluded(key: Field | bigint, message?: string) { + key = Field(key); + + // prove that the key does not exist yet, by showing a valid low node + let low = Provable.witness(Leaf, () => this._findLeaf(key).low); + this.proveInclusion(low, 'Invalid low node (root)'); + low.key.assertLessThan(key, 'Invalid low node (key)'); + key.assertLessThan( + low.nextKey, + message ?? 'Key already exists in the tree' + ); + } + + /** + * Check whether the given key exists in the map. + */ + isIncluded(key: Field | bigint): Bool { + key = Field(key); + + // prove that the key does not exist yet, by showing a valid low node + let low = Provable.witness(Leaf, () => this._findLeaf(key).low); + this.proveInclusion(low, 'Invalid low node (root)'); + low.key.assertLessThan(key, 'Invalid low node (key)'); + key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); + + return low.nextKey.equals(key); + } + // helper methods /** From 5fc02e8f1e806bb297e8b40ac0080df6ab85035c Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 17:40:34 +0200 Subject: [PATCH 55/93] measure constraints --- .../provable/test/merkle-tree.unit-test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 6d2234223d..add6f9e149 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -80,6 +80,25 @@ console.log( ) ); +console.log( + '\nindexed merkle map (assert included)', + constraintSystem.size({ from: [field] }, (key) => { + indexedMap.assertIncluded(key); + }) +); +console.log( + 'indexed merkle map (assert not included)', + constraintSystem.size({ from: [field] }, (key) => { + indexedMap.assertNotIncluded(key); + }) +); +console.log( + 'indexed merkle map (is included)', + constraintSystem.size({ from: [field] }, (key) => { + indexedMap.isIncluded(key); + }) +); + // some manual tests for IndexedMerkleMap { let map = new (IndexedMerkleMap(3))(); From 1c310071cccb565a89fd1a5ca6cbb6de557c6ff4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 18:12:59 +0200 Subject: [PATCH 56/93] prefix low-level / internal APIs with an underscore, some more docs --- src/lib/provable/merkle-tree-indexed.ts | 53 ++++++++++++++----------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index c9f09f7229..8d1a784d9e 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -24,10 +24,10 @@ export { Leaf }; * * let map = new MerkleMap(); * - * map.set(2n, 14n); - * map.set(1n, 13n); + * map.insert(2n, 14n); + * map.insert(1n, 13n); * - * let x = map.get(2n).assertSome(); // 14 + * let x = map.get(2n); // 14 * ``` * * Indexed Merkle maps can be used directly in provable code: @@ -40,7 +40,7 @@ export { Leaf }; * * method(map: MerkleMap, key: Field) { * // get the value associated with `key` - * let value = map.get(key).orElse(0n); + * let value = map.getOption(key).orElse(0n); * * // increment the value by 1 * map.set(key, value.add(1)); @@ -148,7 +148,7 @@ abstract class IndexedMerkleMapAbstract { // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); - this.proveInclusion(low, 'Invalid low node (root)'); + this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); // if the key does exist, we have lowNode.nextKey == key, and this line fails @@ -158,7 +158,7 @@ abstract class IndexedMerkleMapAbstract { // update low node let newLow = { ...low, nextKey: key, nextIndex: index }; - this.root = this.computeRoot(newLow.index, Leaf.hashNode(newLow)); + this.root = this._computeRoot(newLow.index, Leaf.hashNode(newLow)); this._setLeafUnconstrained(true, newLow); // create and append new leaf @@ -169,7 +169,7 @@ abstract class IndexedMerkleMapAbstract { nextIndex: low.nextIndex, }); - this.root = this.computeRoot(indexBits, Leaf.hashNode(leaf)); + this.root = this._computeRoot(indexBits, Leaf.hashNode(leaf)); this.length = this.length.add(1); this._setLeafUnconstrained(false, leaf); } @@ -185,19 +185,24 @@ abstract class IndexedMerkleMapAbstract { // prove that the key exists by presenting a leaf that contains it let self = Provable.witness(Leaf, () => this._findLeaf(key).self); - this.proveInclusion(self, 'Key does not exist in the tree'); + this._proveInclusion(self, 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf (key)'); // at this point, we know that we have a valid update; so we can mutate internal data // update leaf let newSelf = { ...self, value }; - this.root = this.computeRoot(self.index, Leaf.hashNode(newSelf)); + this.root = this._computeRoot(self.index, Leaf.hashNode(newSelf)); this._setLeafUnconstrained(true, newSelf); } /** * Perform _either_ an insertion or update, depending on whether the key exists. + * + * Note: This method is handling both the `insert()` and `update()` case at the same time, so you + * can use it if you don't know whether the key exists or not. + * + * However, this comes at an efficiency cost, so prefer to use `insert()` or `update()` if you know whether the key exists. */ set(key: Field | bigint, value: Field | bigint) { key = Field(key); @@ -205,7 +210,7 @@ abstract class IndexedMerkleMapAbstract { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); - this.proveInclusion(low, 'Invalid low node (root)'); + this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); @@ -217,14 +222,14 @@ abstract class IndexedMerkleMapAbstract { let indexBits = index.toBits(this.height - 1); // prove inclusion of this leaf if it exists - this.proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); + this._proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); // at this point, we know that we have a valid update or insertion; so we can mutate internal data // update low node, or leave it as is let newLow = { ...low, nextKey: key, nextIndex: index }; - this.root = this.computeRoot(low.index, Leaf.hashNode(newLow)); + this.root = this._computeRoot(low.index, Leaf.hashNode(newLow)); this._setLeafUnconstrained(true, newLow); // update leaf, or append a new one @@ -234,7 +239,7 @@ abstract class IndexedMerkleMapAbstract { nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), nextIndex: Provable.if(keyExists, self.nextIndex, low.nextIndex), }); - this.root = this.computeRoot(indexBits, Leaf.hashNode(newLeaf)); + this.root = this._computeRoot(indexBits, Leaf.hashNode(newLeaf)); this.length = Provable.if(keyExists, this.length, this.length.add(1)); this._setLeafUnconstrained(keyExists, newLeaf); } @@ -249,7 +254,7 @@ abstract class IndexedMerkleMapAbstract { // prove that the key exists by presenting a leaf that contains it let self = Provable.witness(Leaf, () => this._findLeaf(key).self); - this.proveInclusion(self, 'Key does not exist in the tree'); + this._proveInclusion(self, 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf (key)'); return self.value; @@ -268,7 +273,7 @@ abstract class IndexedMerkleMapAbstract { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); - this.proveInclusion(low, 'Invalid low node (root)'); + this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); @@ -276,7 +281,7 @@ abstract class IndexedMerkleMapAbstract { let keyExists = low.nextKey.equals(key); // prove inclusion of this leaf if it exists - this.proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); + this._proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); return new OptionField({ isSome: keyExists, value: self.value }); @@ -292,7 +297,7 @@ abstract class IndexedMerkleMapAbstract { // prove that the key exists by presenting a leaf that contains it let self = Provable.witness(Leaf, () => this._findLeaf(key).self); - this.proveInclusion(self, message ?? 'Key does not exist in the tree'); + this._proveInclusion(self, message ?? 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf (key)'); } @@ -304,7 +309,7 @@ abstract class IndexedMerkleMapAbstract { // prove that the key does not exist yet, by showing a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); - this.proveInclusion(low, 'Invalid low node (root)'); + this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThan( low.nextKey, @@ -320,7 +325,7 @@ abstract class IndexedMerkleMapAbstract { // prove that the key does not exist yet, by showing a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); - this.proveInclusion(low, 'Invalid low node (root)'); + this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); @@ -332,20 +337,20 @@ abstract class IndexedMerkleMapAbstract { /** * Helper method to prove inclusion of a leaf in the tree. */ - proveInclusion(leaf: Leaf, message?: string) { + _proveInclusion(leaf: Leaf, message?: string) { // TODO: here, we don't actually care about the index, // so we could add a mode where `computeRoot()` doesn't prove it let node = Leaf.hashNode(leaf); - let root = this.computeRoot(leaf.index, node); + let root = this._computeRoot(leaf.index, node); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); } /** * Helper method to conditionally prove inclusion of a leaf in the tree. */ - proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { + _proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { let node = Leaf.hashNode(leaf); - let root = this.computeRoot(leaf.index, node); + let root = this._computeRoot(leaf.index, node); assert( condition.implies(root.equals(this.root)), message ?? 'Leaf is not included in the tree' @@ -357,7 +362,7 @@ abstract class IndexedMerkleMapAbstract { * * The index can be given as a `Field` or as an array of bits. */ - computeRoot(index: Field | Bool[], node: Field) { + _computeRoot(index: Field | Bool[], node: Field) { let indexBits = index instanceof Field ? index.toBits(this.height - 1) : index; From a7eb3508ec8cca68bdb89be58f155941ee52a18e Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 22:01:21 +0200 Subject: [PATCH 57/93] tweak types, remove abstract from base class --- src/lib/provable/merkle-tree-indexed.ts | 35 ++++++++++++++----------- src/lib/provable/types/unconstrained.ts | 4 +-- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 8d1a784d9e..971126c0fa 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -50,11 +50,17 @@ export { Leaf }; * }) * ``` */ -function IndexedMerkleMap(height: number) { - return class IndexedMerkleMap extends IndexedMerkleMapAbstract { +function IndexedMerkleMap(height: number): typeof IndexedMerkleMapBase { + assert(height > 0, 'height must be positive'); + assert( + height < 53, + 'height must be less than 53, so that we can use 64-bit floats to represent indices.' + ); + + return class IndexedMerkleMap extends IndexedMerkleMapBase { constructor() { // we can't access the abstract `height` property in the base constructor - super(height); + super(); } get height() { @@ -74,13 +80,15 @@ const provableBase = { }), }; -abstract class IndexedMerkleMapAbstract { +class IndexedMerkleMapBase { // data defining the provable interface of a tree root: Field; length: Field; // length of the leaves array // static data defining constraints - abstract get height(): number; + get height() { + return 0; + } // the raw data stored in the tree, plus helper structures readonly data: Unconstrained<{ @@ -97,33 +105,30 @@ abstract class IndexedMerkleMapAbstract { // we'd like to do `abstract static provable` here but that's not supported static provable: Provable< - IndexedMerkleMapAbstract, + IndexedMerkleMapBase, InferValue > = undefined as any; /** * Creates a new, empty Indexed Merkle Map, given its height. */ - constructor(height: number) { - assert(height > 0, 'height must be positive'); - assert( - height < 53, - 'height must be less than 53, so that we can use 64-bit floats to represent indices.' - ); + constructor() { + let height = this.height; let nodes: (bigint | undefined)[][] = Array(height); for (let level = 0; level < height; level++) { nodes[level] = []; } - let firstLeaf = IndexedMerkleMapAbstract._firstLeaf; - let firstNode = Leaf.hashNode(IndexedMerkleMapAbstract._firstLeaf); - let root = Nodes.setLeafNode(nodes, 0, firstNode.toBigInt()); + let firstLeaf = IndexedMerkleMapBase._firstLeaf; + let firstNode = Leaf.hashNode(firstLeaf).toBigInt(); + let root = Nodes.setLeafNode(nodes, 0, firstNode); this.root = Field(root); this.length = Field(1); this.data = Unconstrained.from({ nodes, sortedLeaves: [firstLeaf] }); } + static _firstLeaf = { key: 0n, value: 0n, diff --git a/src/lib/provable/types/unconstrained.ts b/src/lib/provable/types/unconstrained.ts index a6b4baf519..dbdba166f7 100644 --- a/src/lib/provable/types/unconstrained.ts +++ b/src/lib/provable/types/unconstrained.ts @@ -134,11 +134,11 @@ and Provable.asProver() blocks, which execute outside the proof. Unconstrained, Unconstrained > & { - toInput: (x: Unconstrained) => { + toInput: (x: Unconstrained) => { fields?: Field[]; packed?: [Field, number][]; }; - empty: () => Unconstrained; + empty: () => Unconstrained; } { return { ...Unconstrained.provable, From 448c907947c1dde573fe375e7007d72959a50cc6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 22:05:54 +0200 Subject: [PATCH 58/93] fix old version (merkle tree root) --- src/lib/mina/actions/offchain-state-rollup.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index f4cfa6d009..32b91f043d 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -15,7 +15,6 @@ import { MerkleLeaf, updateMerkleMap, } from './offchain-state-serialization.js'; -import { MerkleMap } from '../../provable/merkle-map.js'; import { getProofsEnabled } from '../mina.js'; export { OffchainStateRollup, OffchainStateCommitments }; @@ -28,6 +27,9 @@ class ActionIterator extends MerkleListIterator.create( Actions.emptyActionState() ) {} +const TREE_HEIGHT = 256; +class MerkleMapWitness extends MerkleWitness(TREE_HEIGHT) {} + /** * Commitments that keep track of the current state of an offchain Merkle tree constructed from actions. * Intended to be stored on-chain. @@ -44,7 +46,7 @@ class OffchainStateCommitments extends Struct({ actionState: Field, }) { static empty() { - let emptyMerkleRoot = new MerkleMap().getRoot(); + let emptyMerkleRoot = new MerkleTree(TREE_HEIGHT).getRoot(); return new OffchainStateCommitments({ root: emptyMerkleRoot, actionState: Actions.emptyActionState(), @@ -52,9 +54,6 @@ class OffchainStateCommitments extends Struct({ } } -const TREE_HEIGHT = 256; -class MerkleMapWitness extends MerkleWitness(TREE_HEIGHT) {} - // TODO: it would be nice to abstract the logic for proving a chain of state transition proofs /** From b99ae05fc8a3ffea061a6349800bfd81101133bf Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 22:15:46 +0200 Subject: [PATCH 59/93] make indexed map clonable --- src/lib/provable/merkle-tree-indexed.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 971126c0fa..adb2f5d968 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -103,6 +103,19 @@ class IndexedMerkleMapBase { readonly sortedLeaves: LeafValue[]; }>; + clone() { + let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); + cloned.root = this.root; + cloned.length = this.length; + cloned.data.updateAsProver(({ nodes, sortedLeaves }) => { + return { + nodes: nodes.map((row) => [...row]), + sortedLeaves: [...sortedLeaves], + }; + }); + return cloned; + } + // we'd like to do `abstract static provable` here but that's not supported static provable: Provable< IndexedMerkleMapBase, @@ -559,13 +572,13 @@ class Leaf extends Struct({ } type LeafValue = { - value: bigint; + readonly value: bigint; - key: bigint; - nextKey: bigint; + readonly key: bigint; + readonly nextKey: bigint; - index: bigint; - nextIndex: bigint; + readonly index: bigint; + readonly nextIndex: bigint; }; class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} From d935a7eed92963aac382940c6d483940e0297645 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 22:27:44 +0200 Subject: [PATCH 60/93] return the previous value --- src/lib/provable/merkle-tree-indexed.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index adb2f5d968..d7274a5d5c 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -196,8 +196,10 @@ class IndexedMerkleMapBase { * Update an existing leaf `(key, value)`. * * Proves that the `key` exists. + * + * Returns the previous value. */ - update(key: Field | bigint, value: Field | bigint) { + update(key: Field | bigint, value: Field | bigint): Field { key = Field(key); value = Field(value); @@ -212,6 +214,8 @@ class IndexedMerkleMapBase { let newSelf = { ...self, value }; this.root = this._computeRoot(self.index, Leaf.hashNode(newSelf)); this._setLeafUnconstrained(true, newSelf); + + return self.value; } /** @@ -221,8 +225,10 @@ class IndexedMerkleMapBase { * can use it if you don't know whether the key exists or not. * * However, this comes at an efficiency cost, so prefer to use `insert()` or `update()` if you know whether the key exists. + * + * Returns the previous value, as an option (which is `None` if the key didn't exist before). */ - set(key: Field | bigint, value: Field | bigint) { + set(key: Field | bigint, value: Field | bigint): Option { key = Field(key); value = Field(value); @@ -260,6 +266,9 @@ class IndexedMerkleMapBase { this.root = this._computeRoot(indexBits, Leaf.hashNode(newLeaf)); this.length = Provable.if(keyExists, this.length, this.length.add(1)); this._setLeafUnconstrained(keyExists, newLeaf); + + // return the previous value + return new OptionField({ isSome: keyExists, value: self.value }); } /** From e099ac48ee1d7b3c29db6ecd4bd11cf690667ecf Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 29 May 2024 13:47:32 -0700 Subject: [PATCH 61/93] feat(elliptic-curve.ts): add verifyEcdsaV2 function to support unpacked point tables --- src/lib/provable/gadgets/elliptic-curve.ts | 188 ++++++++++++++++----- 1 file changed, 144 insertions(+), 44 deletions(-) diff --git a/src/lib/provable/gadgets/elliptic-curve.ts b/src/lib/provable/gadgets/elliptic-curve.ts index c8db5b8e03..aec08d80bd 100644 --- a/src/lib/provable/gadgets/elliptic-curve.ts +++ b/src/lib/provable/gadgets/elliptic-curve.ts @@ -212,33 +212,32 @@ function equals(p1: Point, p2: point, Curve: { modulus: bigint }) { return xEquals.and(yEquals); } -/** - * Verify an ECDSA signature. - * - * Details about the `config` parameter: - * - For both the generator point `G` and public key `P`, `config` allows you to specify: - * - the `windowSize` which is used in scalar multiplication for this point. - * this flexibility is good because the optimal window size is different for constant and non-constant points. - * empirically, `windowSize=4` for constants and 3 for variables leads to the fewest constraints. - * our defaults reflect that the generator is always constant and the public key is variable in typical applications. - * - a table of multiples of those points, of length `2^windowSize`, which is used in the scalar multiplication gadget to speed up the computation. - * if these are not provided, they are computed on the fly. - * for the constant G, computing multiples costs no constraints, so passing them in makes no real difference. - * for variable public key, there is a possible use case: if the public key is a public input, then its multiples could also be. - * in that case, passing them in would avoid computing them in-circuit and save a few constraints. - * - The initial aggregator `ia`, see {@link initialAggregator}. By default, `ia` is computed deterministically on the fly. - */ -function verifyEcdsa( +function verifyEcdsaGeneric( Curve: CurveAffine, signature: Ecdsa.Signature, msgHash: Field3, publicKey: Point, + multiScalarMul: ( + scalars: Field3[], + points: Point[], + Curve: CurveAffine, + tableConfigs?: ( + | { + windowSize?: number; + multiples?: Point[]; + } + | undefined + )[], + mode?: 'assert-nonzero' | 'assert-zero', + ia?: point, + hashed?: boolean + ) => Point, config: { G?: { windowSize: number; multiples?: Point[] }; P?: { windowSize: number; multiples?: Point[] }; ia?: point; } = { G: { windowSize: 4 }, P: { windowSize: 4 } } -) { +): Bool { // constant case if ( EcdsaSignature.isConstant(signature) && @@ -285,6 +284,83 @@ function verifyEcdsa( return Provable.equal(Field3.provable, Rx, r); } +/** + * Verify an ECDSA signature. + * + * Details about the `config` parameter: + * - For both the generator point `G` and public key `P`, `config` allows you to specify: + * - the `windowSize` which is used in scalar multiplication for this point. + * this flexibility is good because the optimal window size is different for constant and non-constant points. + * empirically, `windowSize=4` for constants and 3 for variables leads to the fewest constraints. + * our defaults reflect that the generator is always constant and the public key is variable in typical applications. + * - a table of multiples of those points, of length `2^windowSize`, which is used in the scalar multiplication gadget to speed up the computation. + * if these are not provided, they are computed on the fly. + * for the constant G, computing multiples costs no constraints, so passing them in makes no real difference. + * for variable public key, there is a possible use case: if the public key is a public input, then its multiples could also be. + * in that case, passing them in would avoid computing them in-circuit and save a few constraints. + * - The initial aggregator `ia`, see {@link initialAggregator}. By default, `ia` is computed deterministically on the fly. + * @deprecated There is a security vulnerability with this function, allowing a prover to modify witness values than what is expected. Use {@link verifyEcdsaV2} instead. + */ +function verifyEcdsa( + Curve: CurveAffine, + signature: Ecdsa.Signature, + msgHash: Field3, + publicKey: Point, + config: { + G?: { windowSize: number; multiples?: Point[] }; + P?: { windowSize: number; multiples?: Point[] }; + ia?: point; + } = { G: { windowSize: 4 }, P: { windowSize: 4 } } +) { + return verifyEcdsaGeneric( + Curve, + signature, + msgHash, + publicKey, + (scalars, points, Curve, configs, mode, ia) => + multiScalarMul(scalars, points, Curve, configs, mode, ia, true), + config + ); +} + +/** + * Verify an ECDSA signature. + * + * Details about the `config` parameter: + * - For both the generator point `G` and public key `P`, `config` allows you to specify: + * - the `windowSize` which is used in scalar multiplication for this point. + * this flexibility is good because the optimal window size is different for constant and non-constant points. + * empirically, `windowSize=4` for constants and 3 for variables leads to the fewest constraints. + * our defaults reflect that the generator is always constant and the public key is variable in typical applications. + * - a table of multiples of those points, of length `2^windowSize`, which is used in the scalar multiplication gadget to speed up the computation. + * if these are not provided, they are computed on the fly. + * for the constant G, computing multiples costs no constraints, so passing them in makes no real difference. + * for variable public key, there is a possible use case: if the public key is a public input, then its multiples could also be. + * in that case, passing them in would avoid computing them in-circuit and save a few constraints. + * - The initial aggregator `ia`, see {@link initialAggregator}. By default, `ia` is computed deterministically on the fly. + */ +function verifyEcdsaV2( + Curve: CurveAffine, + signature: Ecdsa.Signature, + msgHash: Field3, + publicKey: Point, + config: { + G?: { windowSize: number; multiples?: Point[] }; + P?: { windowSize: number; multiples?: Point[] }; + ia?: point; + } = { G: { windowSize: 4 }, P: { windowSize: 4 } } +) { + return verifyEcdsaGeneric( + Curve, + signature, + msgHash, + publicKey, + (scalars, points, Curve, configs, mode, ia) => + multiScalarMul(scalars, points, Curve, configs, mode, ia, false), + config + ); +} + /** * Bigint implementation of ECDSA verify */ @@ -311,6 +387,36 @@ function verifyEcdsaConstant( return Curve.Scalar.equal(R.x, r); } +function multiScalarMulConstant( + scalars: Field3[], + points: Point[], + Curve: CurveAffine, + mode: 'assert-nonzero' | 'assert-zero' = 'assert-nonzero' +): Point { + let n = points.length; + assert(scalars.length === n, 'Points and scalars lengths must match'); + assertPositiveInteger(n, 'Expected at least 1 point and scalar'); + let useGlv = Curve.hasEndomorphism; + + // TODO dedicated MSM + let s = scalars.map(Field3.toBigint); + let P = points.map(Point.toBigint); + let sum = Curve.zero; + for (let i = 0; i < n; i++) { + if (useGlv) { + sum = Curve.add(sum, Curve.Endo.scale(P[i], s[i])); + } else { + sum = Curve.add(sum, Curve.scale(P[i], s[i])); + } + } + if (mode === 'assert-zero') { + assert(sum.infinity, 'scalar multiplication: expected zero result'); + return Point.from(Curve.zero); + } + assert(!sum.infinity, 'scalar multiplication: expected non-zero result'); + return Point.from(sum); +} + /** * Multi-scalar multiplication: * @@ -339,7 +445,8 @@ function multiScalarMul( | undefined )[] = [], mode: 'assert-nonzero' | 'assert-zero' = 'assert-nonzero', - ia?: point + ia?: point, + hashed: boolean = true ): Point { let n = points.length; assert(scalars.length === n, 'Points and scalars lengths must match'); @@ -348,23 +455,7 @@ function multiScalarMul( // constant case if (scalars.every(Field3.isConstant) && points.every(Point.isConstant)) { - // TODO dedicated MSM - let s = scalars.map(Field3.toBigint); - let P = points.map(Point.toBigint); - let sum = Curve.zero; - for (let i = 0; i < n; i++) { - if (useGlv) { - sum = Curve.add(sum, Curve.Endo.scale(P[i], s[i])); - } else { - sum = Curve.add(sum, Curve.scale(P[i], s[i])); - } - } - if (mode === 'assert-zero') { - assert(sum.infinity, 'scalar multiplication: expected zero result'); - return Point.from(Curve.zero); - } - assert(!sum.infinity, 'scalar multiplication: expected non-zero result'); - return Point.from(sum); + return multiScalarMulConstant(scalars, points, Curve, mode); } // parse or build point tables @@ -444,14 +535,22 @@ function multiScalarMul( if (i % windowSize === 0) { // pick point to add based on the scalar chunk let sj = scalarChunks[j][i / windowSize]; - let sjP = - windowSize === 1 - ? points[j] - : arrayGetGeneric( - HashedPoint.provable, - hashedTables[j], - sj - ).unhash(); + let sjP; + if (hashed) { + sjP = + windowSize === 1 + ? points[j] + : arrayGetGeneric( + HashedPoint.provable, + hashedTables[j], + sj + ).unhash(); + } else { + sjP = + windowSize === 1 + ? points[j] + : arrayGetGeneric(Point.provable, tables[j], sj); + } // ec addition let added = add(sum, sjP, Curve); @@ -725,6 +824,7 @@ const EcdsaSignature = { const Ecdsa = { sign: signEcdsa, verify: verifyEcdsa, + verifyV2: verifyEcdsaV2, Signature: EcdsaSignature, }; From 9fa198e6c58121913361e17b692a41dbeed31b6a Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 29 May 2024 13:54:16 -0700 Subject: [PATCH 62/93] feat(ecdsa.ts, foreign-ecdsa.ts, custom-gates-recursion.unit-test.ts, ecdsa.unit-test.ts): replace deprecated verify and verifySignedHash methods with verifyV2 and verifySignedHashV2 to address security vulnerability --- src/examples/crypto/ecdsa/ecdsa.ts | 4 +- src/lib/provable/crypto/foreign-ecdsa.ts | 63 +++++++++++++++++++ .../test/custom-gates-recursion.unit-test.ts | 2 +- src/lib/provable/test/ecdsa.unit-test.ts | 33 +++++++++- 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/src/examples/crypto/ecdsa/ecdsa.ts b/src/examples/crypto/ecdsa/ecdsa.ts index 8e9e951ee6..986a0f2522 100644 --- a/src/examples/crypto/ecdsa/ecdsa.ts +++ b/src/examples/crypto/ecdsa/ecdsa.ts @@ -23,7 +23,7 @@ const keccakAndEcdsa = ZkProgram({ verifyEcdsa: { privateInputs: [Ecdsa.provable, Secp256k1.provable], async method(message: Bytes32, signature: Ecdsa, publicKey: Secp256k1) { - return signature.verify(message, publicKey); + return signature.verifyV2(message, publicKey); }, }, }, @@ -38,7 +38,7 @@ const ecdsa = ZkProgram({ verifySignedHash: { privateInputs: [Ecdsa.provable, Secp256k1.provable], async method(message: Scalar, signature: Ecdsa, publicKey: Secp256k1) { - return signature.verifySignedHash(message, publicKey); + return signature.verifySignedHashV2(message, publicKey); }, }, }, diff --git a/src/lib/provable/crypto/foreign-ecdsa.ts b/src/lib/provable/crypto/foreign-ecdsa.ts index b601928b93..fdef048a0f 100644 --- a/src/lib/provable/crypto/foreign-ecdsa.ts +++ b/src/lib/provable/crypto/foreign-ecdsa.ts @@ -99,6 +99,7 @@ class EcdsaSignature { * let isValid = sig.verify(msg, pk); * isValid.assertTrue('signature verifies'); * ``` + * @deprecated There is a security vulnerability in this method. Use {@link verifyV2} instead. */ verify(message: Bytes, publicKey: FlexiblePoint) { let msgHashBytes = Keccak.ethereum(message); @@ -106,12 +107,53 @@ class EcdsaSignature { return this.verifySignedHash(msgHash, publicKey); } + /** + * Verify the ECDSA signature given the message (an array of bytes) and public key (a {@link Curve} point). + * + * **Important:** This method returns a {@link Bool} which indicates whether the signature is valid. + * So, to actually prove validity of a signature, you need to assert that the result is true. + * + * @throws if one of the signature scalars is zero or if the public key is not on the curve. + * + * @example + * ```ts + * // create classes for your curve + * class Secp256k1 extends createForeignCurve(Crypto.CurveParams.Secp256k1) {} + * class Scalar extends Secp256k1.Scalar {} + * class Ecdsa extends createEcdsa(Secp256k1) {} + * + * let message = 'my message'; + * let messageBytes = new TextEncoder().encode(message); + * + * // outside provable code: create inputs + * let privateKey = Scalar.random(); + * let publicKey = Secp256k1.generator.scale(privateKey); + * let signature = Ecdsa.sign(messageBytes, privateKey.toBigInt()); + * + * // ... + * // in provable code: create input witnesses (or use method inputs, or constants) + * let pk = Provable.witness(Secp256k1.provable, () => publicKey); + * let msg = Provable.witness(Provable.Array(Field, 9), () => messageBytes.map(Field)); + * let sig = Provable.witness(Ecdsa.provable, () => signature); + * + * // verify signature + * let isValid = sig.verify(msg, pk); + * isValid.assertTrue('signature verifies'); + * ``` + */ + verifyV2(message: Bytes, publicKey: FlexiblePoint) { + let msgHashBytes = Keccak.ethereum(message); + let msgHash = keccakOutputToScalar(msgHashBytes, this.Constructor.Curve); + return this.verifySignedHashV2(msgHash, publicKey); + } + /** * Verify the ECDSA signature given the message hash (a {@link Scalar}) and public key (a {@link Curve} point). * * This is a building block of {@link EcdsaSignature.verify}, where the input message is also hashed. * In contrast, this method just takes the message hash (a curve scalar) as input, giving you flexibility in * choosing the hashing algorithm. + * @deprecated There is a security vulnerability in this method. Use {@link verifySignedHashV2} instead. */ verifySignedHash( msgHash: AlmostForeignField | bigint, @@ -127,6 +169,27 @@ class EcdsaSignature { ); } + /** + * Verify the ECDSA signature given the message hash (a {@link Scalar}) and public key (a {@link Curve} point). + * + * This is a building block of {@link EcdsaSignature.verify}, where the input message is also hashed. + * In contrast, this method just takes the message hash (a curve scalar) as input, giving you flexibility in + * choosing the hashing algorithm. + */ + verifySignedHashV2( + msgHash: AlmostForeignField | bigint, + publicKey: FlexiblePoint + ) { + let msgHash_ = this.Constructor.Curve.Scalar.from(msgHash); + let publicKey_ = this.Constructor.Curve.from(publicKey); + return Ecdsa.verifyV2( + this.Constructor.Curve.Bigint, + toObject(this), + msgHash_.value, + toPoint(publicKey_) + ); + } + /** * Create an {@link EcdsaSignature} by signing a message with a private key. * diff --git a/src/lib/provable/test/custom-gates-recursion.unit-test.ts b/src/lib/provable/test/custom-gates-recursion.unit-test.ts index 26a9e7d33f..7d1a06eb55 100644 --- a/src/lib/provable/test/custom-gates-recursion.unit-test.ts +++ b/src/lib/provable/test/custom-gates-recursion.unit-test.ts @@ -46,7 +46,7 @@ let program = ZkProgram({ let msgHash_ = Provable.witness(Field3.provable, () => msgHash); let publicKey_ = Provable.witness(Point.provable, () => publicKey); - return Ecdsa.verify(Secp256k1, signature_, msgHash_, publicKey_); + return Ecdsa.verifyV2(Secp256k1, signature_, msgHash_, publicKey_); }, }, }, diff --git a/src/lib/provable/test/ecdsa.unit-test.ts b/src/lib/provable/test/ecdsa.unit-test.ts index 97e9b93f30..89545d9645 100644 --- a/src/lib/provable/test/ecdsa.unit-test.ts +++ b/src/lib/provable/test/ecdsa.unit-test.ts @@ -75,7 +75,7 @@ for (let Curve of curves) { let hasGlv = Curve.hasEndomorphism; if (noGlv) Curve.hasEndomorphism = false; // hack to force non-GLV version try { - return Ecdsa.verify(Curve, sig.signature, sig.msg, sig.publicKey); + return Ecdsa.verifyV2(Curve, sig.signature, sig.msg, sig.publicKey); } finally { Curve.hasEndomorphism = hasGlv; } @@ -140,7 +140,7 @@ let msgHash = const ia = initialAggregator(Secp256k1); const config = { G: { windowSize: 4 }, P: { windowSize: 4 }, ia }; -let program = ZkProgram({ +let deprecatedVerify = ZkProgram({ name: 'ecdsa', publicOutput: Bool, methods: { @@ -166,6 +166,32 @@ let program = ZkProgram({ }, }); +let program = ZkProgram({ + name: 'ecdsa', + publicOutput: Bool, + methods: { + ecdsa: { + privateInputs: [], + async method() { + let signature_ = Provable.witness( + Ecdsa.Signature.provable, + () => signature + ); + let msgHash_ = Provable.witness(Field3.provable, () => msgHash); + let publicKey_ = Provable.witness(Point.provable, () => publicKey); + + return Ecdsa.verifyV2( + Secp256k1, + signature_, + msgHash_, + publicKey_, + config + ); + }, + }, + }, +}); + console.time('ecdsa verify (constant)'); program.rawMethods.ecdsa(); console.timeEnd('ecdsa verify (constant)'); @@ -182,11 +208,14 @@ console.log(cs.summary()); console.time('ecdsa verify (compile)'); await program.compile(); +await deprecatedVerify.compile(); console.timeEnd('ecdsa verify (compile)'); console.time('ecdsa verify (prove)'); let proof = await program.ecdsa(); +let proof2 = await deprecatedVerify.ecdsa(); console.timeEnd('ecdsa verify (prove)'); assert(await program.verify(proof), 'proof verifies'); +assert(await deprecatedVerify.verify(proof2), 'deprecated proof verifies'); proof.publicOutput.assertTrue('signature verifies'); From c1b6c3730ff292cd786f2fe9444c7aba08df1bcc Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 29 May 2024 13:57:10 -0700 Subject: [PATCH 63/93] chore(vk-regression.json): update vk test artifact --- tests/vk-regression/vk-regression.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/vk-regression/vk-regression.json b/tests/vk-regression/vk-regression.json index 3b4a33c0a6..d80e7a6fc8 100644 --- a/tests/vk-regression/vk-regression.json +++ b/tests/vk-regression/vk-regression.json @@ -262,29 +262,29 @@ } }, "ecdsa-only": { - "digest": "10f4849c3f8b361abf19c0fedebb445d98f039c137f48e2b130110568e2054a3", + "digest": "3e306cbba2e987fe302992a81aba9551d47df260f7a3b8b8abeee6bc938d7", "methods": { "verifySignedHash": { - "rows": 29105, - "digest": "6949309a2e8aa6b73e297785c8d3c1bd" + "rows": 34257, + "digest": "0d84b4072ac7aeb3f387d1b03b31809f" } }, "verificationKey": { - "data": "AAD0TiJIvE46IEuFjZed3VZt7S8Wp0kRCJXb3a2Ju7WUBunBgHb4SXvz1c3QjP2nd1qSYoUr66taz9IKVgu+5so8TiBnYTRcd7RsaAjbdbIbQJ9EuopFRFewZRx9qeQeEibNeMRcRMP4LdfS3AQRxhFZzN4HFa4MbtGs+Aja820cIw5azWYCxKf0pWJRDHbLUtJrjnQT0ATHD77rtbgLedcfFKfCQS5oAbF7hIhCbAsm7wMT+9+ZX8M5354UKZ03NiLUh5XNLAlGgVi7FfWR6p9P72AAymyD3lUdecJyZmCREiVgPrTdFppkp45TefJWNTySkV9c5YzpNxQoXedZDvYP/5s4KBkfIeK+zB2yJC9eZ1ZDYfM88shGDYxmBtur9AkQ49QGquR+kYUI0lpXtuNMG+ZRy0FRJ8ci/TE+PIPIFnSiGcSOA3YM2G171LYf89abU2QUoQRHSP3PmmAOy/8CoRLVro7Nl6z/Ou0oZzX7RjOEo//LBqcSWa2S9X8TQz0R3uivbovTdq0rrba56SbEnK6LWItmBc6CubYWL7UzDbD3RZM6iRz1hqTHDzDz7UIWOzHgLqW9rjnZllQCyfsSAG8m+uV+FE9u7B5HDeV/lcncoeZ+3W07KuxIDERk2Rkw9R/nhpXJfdp/CHOvQgDi3sWzS5mFnuqOMgimer4b2jkgKzcNZhhPW5VfbcSYDpx5nVaU5pTEFl+2+RlcuhBpG1ksAWbD64AUKDjdyTWIC5Wn68AagPtG65V13eFS5LgkSfVtNXxGodg7SdP4AJmXpBgZfzMg4RW6Qje5ZFfrwRzoHPo0y7nO1hkaNLGV3Wvd3/pYiebXvyo+DdTZmaMbJpJaGSCysnovOrVUIpcn4h1hvA12jztQFQcbNHoVeZgslPxA54y9ynjhN7VZfT8lNXXIrRCpmPaxZW6Bw6Op/g6FGIs8TlruzZJRhz1lOLvl2FPvrUsFtz4yWTPjbT+VGsKuJFPvMuYybxq8pGyWVQN023uObDel47krlcQoH4MAdc0EQMBqCWPmRrqG9/IrZC3yjRQ8Tp9pFhNqWbwWmDTYHp1A/7xjPFP/wrrOeUYMlmQbtSp/XYZxqCYL4fOsOvDR5DLlenSa0wQ3PXdv/C9LpDvkzJOLZs+/ZePd4YMI0+WuP2+6Xas4aNM+4JkNuHF5uMDcxgWID4TUy7Vdlzm3CVbhX15uBoKhuYWQgLr2rnVJ5SOZoDvlwJtcK2izLMYVAasejw4fvsehYGb88wvDbFxS6sM9gDSgTlavZRs95Qf+c1KpYf/jb8BxYNrwrqy8F++c1APDzfzQ/IbVLiaL28wkEy412qmXSjM+9hErKXFy8JIT/WBOIWMMg/7mMjvxngHnci+aYJZ6J+Lszh5zgo708vzO7fwaxC0wgd8anH3gFrbFnOg1hkmmoUEIgIwXh+ynuoZPOaoKNXNm1jOl8HpdFOG7vpQavC600YgzS2YGtY7K2WQ5GtN5ZTZBHPsUSir2yKSo9Le9CWXbDtn3SBDepWypwDa3YWKtNog+y10VmpL1N+RG3u1DXSuY7y9WZgkQ7tdvyx/Gjr91kjF0s3bt7vHIAZCtzNlRlWDBz3og0cSnEucCEuKR6dL2Mz+RuF1GmLoXZXapUjVG/82BjdAMAOxPlE67lEs+JWgnrVrA5NLJoL4DZ6+fhQKpNfk0uOrEfZIWR9Sau0IBwBxu6IYVm5/XAB19dt8MAuVcRdN/JGGzo0Hr3WVJuKzbAhuFwJZzcd1J1n4xO09ECT5NQdFSFXGsy8kIFjRNEOkLl+bAExePtGCt0w6cYqB0uCeX3lTI7ugIEgdStMtHFiWngJ218l8CuVrkwTJ7ZqHLtuJDiNqlLptkHWChDfw+IgDwz85dZrfBBzQrMRWranxQmisM+wx3vC+pLURRQHZJEasGCAElj0lTColrqQ/cXS7cBaqs1tBsQDGzKYMCMwsqL53fyxGCljVvljBa99+FpYfoUK+Fi0z6uEbem+luXRScr2yPB5I08lnBY23RmBb/pfSyBfbcmnmF5BkRlJTJKY7fQL/t9bFfywoquQe9e7OQvIjppA/FO7HmZS6hoOU+eS8+W94fEF2gvrowpTeqQHM6hLN9Qzl8niwZWUIyRCfyuzQnuSz/VP1K2sMFBKnZZNDcuBh1/xSFymOH6LfNKostvc6qHTIxrTjlH6952bo1bQl+mVvBUaJuRkYh12QbcyIyzcBFUYwaFazzkHXMof0O30oL3Q6wegTvJxTSZD5VCr5D26Myzoa0JBpqL0st9/MNGZe5a/+HW1qan/VtGA5nYkJcUzwKVqqlmZeuOZekFLGxlfp0lv9IQUQWtiU5uvd5HVoolEc/teUnx/IxYe01IDxX9cbmPMJnLYXJGSY=", - "hash": "8976008648274552011517161147643684183059283822571100321599783980624376784594" + "data": "AAD0TiJIvE46IEuFjZed3VZt7S8Wp0kRCJXb3a2Ju7WUBunBgHb4SXvz1c3QjP2nd1qSYoUr66taz9IKVgu+5so8TiBnYTRcd7RsaAjbdbIbQJ9EuopFRFewZRx9qeQeEibNeMRcRMP4LdfS3AQRxhFZzN4HFa4MbtGs+Aja820cIw5azWYCxKf0pWJRDHbLUtJrjnQT0ATHD77rtbgLedcfFKfCQS5oAbF7hIhCbAsm7wMT+9+ZX8M5354UKZ03NiLUh5XNLAlGgVi7FfWR6p9P72AAymyD3lUdecJyZmCREiVgPrTdFppkp45TefJWNTySkV9c5YzpNxQoXedZDvYP/5s4KBkfIeK+zB2yJC9eZ1ZDYfM88shGDYxmBtur9AkQ49QGquR+kYUI0lpXtuNMG+ZRy0FRJ8ci/TE+PIPIFnSiGcSOA3YM2G171LYf89abU2QUoQRHSP3PmmAOy/8CoRLVro7Nl6z/Ou0oZzX7RjOEo//LBqcSWa2S9X8TQz0R3uivbovTdq0rrba56SbEnK6LWItmBc6CubYWL7UzDbD3RZM6iRz1hqTHDzDz7UIWOzHgLqW9rjnZllQCyfsSAAs+mZNp6eMcA9nsWiKZeNIjKS/efiDZVKA0XR4LNUoAy2KsbYbk7XBknDhh0/POjv9ThJ81GTAksHfHEjyjPAcgKzcNZhhPW5VfbcSYDpx5nVaU5pTEFl+2+RlcuhBpG1ksAWbD64AUKDjdyTWIC5Wn68AagPtG65V13eFS5LgkSfVtNXxGodg7SdP4AJmXpBgZfzMg4RW6Qje5ZFfrwRzoHPo0y7nO1hkaNLGV3Wvd3/pYiebXvyo+DdTZmaMbJpJaGSCysnovOrVUIpcn4h1hvA12jztQFQcbNHoVeZgslPxA54y9ynjhN7VZfT8lNXXIrRCpmPaxZW6Bw6Op/g6P1Y8pKZHixBy1UrxqWGI+49oRtRFGw9CWS21EekuBFeu9RKI6yZLDiyRC2b3koFG+Kp6oq5Ej6Q8uargE09Ago9Z2Ak+8OPG2V17zCdK2baF3CgW/HuWQT3dmWCesPz2nk99eNiSJzx2lLOFaWlp8VCG02On55KRZInJzhIEMP146ddCU9VJ1JmYsYa+MYEgKjZCvABbX9AEY2ggMr1cHaA49GrGul+Sj6pAvz4oyzaR8m7WAPMDuBtVwdbDtfju3CVbhX15uBoKhuYWQgLr2rnVJ5SOZoDvlwJtcK2izLMYVAasejw4fvsehYGb88wvDbFxS6sM9gDSgTlavZRs95Qf+c1KpYf/jb8BxYNrwrqy8F++c1APDzfzQ/IbVLiaL28wkEy412qmXSjM+9hErKXFy8JIT/WBOIWMMg/7mMu1Al7Tt/kOZrDznlS/szLlpAp2jISa8VWCmlAEPrustdNqQvptSsF6hikzXZVXg5f8pU4Gpa0TP0TRFvIYfmTyl8HpdFOG7vpQavC600YgzS2YGtY7K2WQ5GtN5ZTZBHPsUSir2yKSo9Le9CWXbDtn3SBDepWypwDa3YWKtNog+y10VmpL1N+RG3u1DXSuY7y9WZgkQ7tdvyx/Gjr91kjF0s3bt7vHIAZCtzNlRlWDBz3og0cSnEucCEuKR6dL2Mz+RuF1GmLoXZXapUjVG/82BjdAMAOxPlE67lEs+JWgnrVrA5NLJoL4DZ6+fhQKpNfk0uOrEfZIWR9Sau0IBwBxu6IYVm5/XAB19dt8MAuVcRdN/JGGzo0Hr3WVJuKzbAhuFwJZzcd1J1n4xO09ECT5NQdFSFXGsy8kIFjRNEOkLl+bAExePtGCt0w6cYqB0uCeX3lTI7ugIEgdStMtHFiWngJ218l8CuVrkwTJ7ZqHLtuJDiNqlLptkHWChDfw+IgDwz85dZrfBBzQrMRWranxQmisM+wx3vC+pLURRQHZJEasGCAElj0lTColrqQ/cXS7cBaqs1tBsQDGzKYMCMwsqL53fyxGCljVvljBa99+FpYfoUK+Fi0z6uEbem+luXRScr2yPB5I08lnBY23RmBb/pfSyBfbcmnmF5BkRlJTJKY7fQL/t9bFfywoquQe9e7OQvIjppA/FO7HmZS6hoOU+eS8+W94fEF2gvrowpTeqQHM6hLN9Qzl8niwZWUIyRCfyuzQnuSz/VP1K2sMFBKnZZNDcuBh1/xSFymOH6LfNKostvc6qHTIxrTjlH6952bo1bQl+mVvBUaJuRkYh12QbcyIyzcBFUYwaFazzkHXMof0O30oL3Q6wegTvJxTSZD5VCr5D26Myzoa0JBpqL0st9/MNGZe5a/+HW1qan/VtGA5nYkJcUzwKVqqlmZeuOZekFLGxlfp0lv9IQUQWtiU5uvd5HVoolEc/teUnx/IxYe01IDxX9cbmPMJnLYXJGSY=", + "hash": "21569291188270991858171229626958471686538549641430686923686948193105860016913" } }, "ecdsa": { - "digest": "3db104818bb676cfa0cbf19e1f3d0bfa43ecf6694a1c9c72414db8dcd8cbdea7", + "digest": "d3869b477c270fdef32c71260bfca827f8840666b0d178a5c70606013038411", "methods": { "verifyEcdsa": { - "rows": 43603, - "digest": "620578262a508a25f0562bbf7757f3f2" + "rows": 48755, + "digest": "4c8bf67689c0247a32385f3d971ff6ce" } }, "verificationKey": { - "data": "AAClzwJllhsXW4UHH6dHRfysOr2Bv96SCGeRT4Wa3IseIaKiigrgpOaDXTEEjlaJI+fiakjF1+4p1Q0TFxwpf9QVFNjCZ84EnGkie609NhFB8tU9k5Vkoqw3jihdsoJEUy6GK0H30dl/7H1rGxsx6Ec05aaFhiPw6t0jLxF1kj4uIUUtyjzEBEpi1D8Jgw5ToAzE/dLLvgPlMu+gZu6pyyoQxnJ/0SYRd63N82MLWgTkbP8yzNSL5FFaqjZtE5VjvjZin4GWyDB9279EZ6D6avFW2l7WuMJG++xBqGsNKZUgNM4WkUGNfCd+m42hJgt46eOy89db672su0n24IZG9tAsgQl8vPsVKfsTvTWlMj6/jISm7Dcctr1rZpSb8hRPsQstlfqMw3q6qijtTkFiMsdGRwJ6LNukSFUxOarhVsfREQngJufm4IxFpJJMR5F1DFSDPiOPuylEqXzke+j078Y4vr+QRo07YRlsoEv4a6ChcxMd3uu5Oami+D747/YVaS8kLd/3bO+WFpubID5fv4F7/JO4Fy/O7n1waPpNnzi/PZRlHVlwzNVAs09OmTmgzNM4/jAJBO9lRgCFA1SW0BADAB/OjbqHwZEIe/gSZHjdf9LoENxdzScMgwNzrhV5AisR7jG4jHIGeEZiHE8zWbiLW6lhwX72EOSI0oJES29MYgTPFNzYqZw3swyXzQ3nvZqWU2ARuzo1BgMrvnDgW1H+AMbKbNGU7IYXIYaLfTR9S7qrUbHESHac4wo9J9HmRiU1/IQdyr5LldYkzYtZOrjM4SzBkYYVtpSH7Sopij/TTy0U9CXNle7iCnZQS/72C8kwyJ+BGqpULLkSWhHoj+U9GSW9UgDHZ62jRTzvuZz5QaX/hYOmpNChNMFS1zoDYVE7ZIzVQKX03IDkzHAVJCXggwhQO3NK6OGhlP7A/heM6zgiR3/LlOa8uW4fcow50XC3280SDziK0Uczab3zlYXPPH6KqGPJfnftgwuvcHsRgddOWDVfEH3Q9mAj0y1R1Fop80NdjZzoTTIs/44QSWiFyG39nQX8v3NMpsd6zueZERUWffAvqTtKasyT1oruKKvgjsTTHmCH0v2zspYAsJpTMQ2qDXacvJQHRIiBHfPZ3G52Z2lTf6OGg/elBurqGhA2wdDAQrBIWJwiTClONbV+8yR/4Md7aPi44E4XICLpHhE5hzko7ePy9cwh3oXy3btBt0urRwrl4d/jhHvoYt1eE2inNWEOYdlkXFUDlDErwOpFVsyQon0G25zNLAcVaZgdJLWueU1y3G0XkfHRqMZ8eV1iNAegPCCNRCvJ6SVsSwcQ67s45a8VqFxSSW0F65bDCI6Ue3Hwpb1RFKbfSIJbPyUrVSq5K99wUJ01O93Kn8LQlrAbjHWo5Za+tW0a/+Qlbr5E2eSEge+ldnbMbA9rcJwZf4bT457dBXMdlD7mECIDZtD8M/KLeyzMEinDzPfqnwZjU2ifxs6gaJPXOQAWPzbCm/z2vGlRbXDGZF6yTbLTdjzviuPhVtb7bzsZW2AYC+TlZqb4qm9MAVsH5rX3OZmvvmw5oRKeSj+FFD7uSRwfutDGC99i93uptU8syL/8Tr8xU3atxITlSqHqG+rVGWdLO9i3iq38zXgXbvZacrc3CMF5QBIM8yZXNslXH5k39D5SqubSHBWTqAJ1I0heOjaIHQGLROBYLn178tckBxfKQ2UpyfkvMw1Waw+fp5f64Ce+5bmYyZr6Dhmw/xcoAihjUsEqoecrLuGPp6qI4hQt9qOnVrAxHzwwtJGxcqoiCbe1mgz0fxMCt/i0z3ygdqAn20DKPHuBdqgVUFwx2T7Ac9fUCf3RHMq34onrr2nLHc038GYedmlFjoUZStujGwA8tSwLWyuWZTDVV+ZaW92qkhmrACog6NwhR6SEjQgsMRCVBQZzYirZxyulYmcNWH6BUmnLLFsn3GbS40xUr70gujEPnjZUK/ExGRfUPOfrYYb8mAciE9nP8OeK/UI+zjJy6Qp8mMroFw7gVHCfDtKTeQFt4JV3zubGsD7jypquHKCqPewhgn9tZ1UIsKIQB7+hBwDHzhlOZ2FfR4eLwQkO8sz275tpjHDAqX/TBWWRVg/yBDii0CWN4bP8UuX36jZKZboJUxIkM1xThiGZM2/oMbe5cZyjgrBR3P21wiDHAAlsHkaMfJgkVLqvZOw8hflKRIMa2dEYo5voD6aV30sATHQLoV0o+MlV3WA38RA+23Jqt1g+UZ7ReAuDP88jXhqWFcIvWHrJG0oy+rpAPQU/38vhIxbl//lirsirdVK2LrU47CC1f9/pRi07vTnvAm+n02dhwriqpwOmI2o2OU4mO0q96pCueKjAttkXgz+NSIJzcwprvNyE9UtKWswmIQg=", - "hash": "16302996689470245844573621505321306831143065228180586982555898856032378210296" + "data": "AAClzwJllhsXW4UHH6dHRfysOr2Bv96SCGeRT4Wa3IseIaKiigrgpOaDXTEEjlaJI+fiakjF1+4p1Q0TFxwpf9QVFNjCZ84EnGkie609NhFB8tU9k5Vkoqw3jihdsoJEUy6GK0H30dl/7H1rGxsx6Ec05aaFhiPw6t0jLxF1kj4uIUUtyjzEBEpi1D8Jgw5ToAzE/dLLvgPlMu+gZu6pyyoQxnJ/0SYRd63N82MLWgTkbP8yzNSL5FFaqjZtE5VjvjZin4GWyDB9279EZ6D6avFW2l7WuMJG++xBqGsNKZUgNM4WkUGNfCd+m42hJgt46eOy89db672su0n24IZG9tAsgQl8vPsVKfsTvTWlMj6/jISm7Dcctr1rZpSb8hRPsQstlfqMw3q6qijtTkFiMsdGRwJ6LNukSFUxOarhVsfREQngJufm4IxFpJJMR5F1DFSDPiOPuylEqXzke+j078Y4vr+QRo07YRlsoEv4a6ChcxMd3uu5Oami+D747/YVaS8kLd/3bO+WFpubID5fv4F7/JO4Fy/O7n1waPpNnzi/PZRlHVlwzNVAs09OmTmgzNM4/jAJBO9lRgCFA1SW0BADAG+OYsDBncznvyv8yKo/oM0S2E69D1NQ9l9E8fNGwNgQNP+1l2h5pLgBVt3LKl6gEII2VyxbcDkGJMFYcvk8QhPPFNzYqZw3swyXzQ3nvZqWU2ARuzo1BgMrvnDgW1H+AMbKbNGU7IYXIYaLfTR9S7qrUbHESHac4wo9J9HmRiU1/IQdyr5LldYkzYtZOrjM4SzBkYYVtpSH7Sopij/TTy0U9CXNle7iCnZQS/72C8kwyJ+BGqpULLkSWhHoj+U9GSW9UgDHZ62jRTzvuZz5QaX/hYOmpNChNMFS1zoDYVE7ZIzVQKX03IDkzHAVJCXggwhQO3NK6OGhlP7A/heM6zgiR3/LlOa8uW4fcow50XC3280SDziK0Uczab3zlYXPPH6KqGPJfnftgwuvcHsRgddOWDVfEH3Q9mAj0y1R1Fop+NI/+Rbg+iznL7GXHPnFLngtJb9NbC+hwyed49ABOTzAwt4Y2rXZ3XDuDalHoWlr1oQ1bbZK7/b1zuecVAElIQ2qDXacvJQHRIiBHfPZ3G52Z2lTf6OGg/elBurqGhA2wdDAQrBIWJwiTClONbV+8yR/4Md7aPi44E4XICLpHhE5hzko7ePy9cwh3oXy3btBt0urRwrl4d/jhHvoYt1eE2inNWEOYdlkXFUDlDErwOpFVsyQon0G25zNLAcVaZgdJLWueU1y3G0XkfHRqMZ8eV1iNAegPCCNRCvJ6SVsSwcQ67s45a8VqFxSSW0F65bDCI6Ue3Hwpb1RFKbfSIJbPyUrVSq5K99wUJ01O93Kn8LQlrAbjHWo5Za+tW0a/+Qlbr5E2eSEge+ldnbMbA9rcJwZf4bT457dBXMdlD7mECIDZtD8M/KLeyzMEinDzPfqnwZjU2ifxs6gaJPXOQAWPzbCm/z2vGlRbXDGZF6yTbLTdjzviuPhVtb7bzsZW2AYC+TlZqb4qm9MAVsH5rX3OZmvvmw5oRKeSj+FFD7uSRwfutDGC99i93uptU8syL/8Tr8xU3atxITlSqHqG+rVGWdLO9i3iq38zXgXbvZacrc3CMF5QBIM8yZXNslXH5k39D5SqubSHBWTqAJ1I0heOjaIHQGLROBYLn178tckBxfKQ2UpyfkvMw1Waw+fp5f64Ce+5bmYyZr6Dhmw/xcoAihjUsEqoecrLuGPp6qI4hQt9qOnVrAxHzwwtJGxcqoiCbe1mgz0fxMCt/i0z3ygdqAn20DKPHuBdqgVUFwx2T7Ac9fUCf3RHMq34onrr2nLHc038GYedmlFjoUZStujGwA8tSwLWyuWZTDVV+ZaW92qkhmrACog6NwhR6SEjQgsMRCVBQZzYirZxyulYmcNWH6BUmnLLFsn3GbS40xUr70gujEPnjZUK/ExGRfUPOfrYYb8mAciE9nP8OeK/UI+zjJy6Qp8mMroFw7gVHCfDtKTeQFt4JV3zubGsD7jypquHKCqPewhgn9tZ1UIsKIQB7+hBwDHzhlOZ2FfR4eLwQkO8sz275tpjHDAqX/TBWWRVg/yBDii0CWN4bP8UuX36jZKZboJUxIkM1xThiGZM2/oMbe5cZyjgrBR3P21wiDHAAlsHkaMfJgkVLqvZOw8hflKRIMa2dEYo5voD6aV30sATHQLoV0o+MlV3WA38RA+23Jqt1g+UZ7ReAuDP88jXhqWFcIvWHrJG0oy+rpAPQU/38vhIxbl//lirsirdVK2LrU47CC1f9/pRi07vTnvAm+n02dhwriqpwOmI2o2OU4mO0q96pCueKjAttkXgz+NSIJzcwprvNyE9UtKWswmIQg=", + "hash": "1104432843216666147630995603505954039049338382620925003943517086518984015452" } }, "sha256": { From 21f6ebd63c988b0c0d6818102f83e9caf3b3f440 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 29 May 2024 14:00:22 -0700 Subject: [PATCH 64/93] docs(CHANGELOG.md): update changelog with deprecated and new methods in Ecdsa class due to security vulnerability fix and enhancement --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac3634036d..6537be4756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,12 +23,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - This can pose an attack surface, since it is easy to maliciously pick either the `+0` or the `-0` representation - Use `Int64.isPositiveV2()` and `Int64.modV2()` instead - Also deprecated `Int64.neg()` in favor of `Int64.negV2()`, for compatibility with v2 version of `Int64` that will use `Int64.checkV2()` +- `Ecdsa.verify()` and `Ecdsa.verifySignedHash()` deprecated in favor of `Ecdsa.verifyV2()` and `Ecdsa.verifySignedHashV2()` due to a security vulnerability found in the current implementation https://github.com/o1-labs/o1js/pull/1669 ## [1.3.0](https://github.com/o1-labs/o1js/compare/6a1012162...54d6545bf) ### Added - Added `base64Encode()` and `base64Decode(byteLength)` methods to the `Bytes` class. https://github.com/o1-labs/o1js/pull/1659 +- Added `Ecdsa.verifyV2()` and `Ecdsa.verifySignedHashV2` methods to the `Ecdsa` class. https://github.com/o1-labs/o1js/pull/1669 ### Fixes From 2c3f1a177165ae67e2f4d0aeaa8dfb876f37aeac Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 11:19:01 +0200 Subject: [PATCH 65/93] minor --- src/lib/provable/merkle-tree-indexed.ts | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 971126c0fa..23f4d6e8cd 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -122,7 +122,7 @@ class IndexedMerkleMapBase { let firstLeaf = IndexedMerkleMapBase._firstLeaf; let firstNode = Leaf.hashNode(firstLeaf).toBigInt(); - let root = Nodes.setLeafNode(nodes, 0, firstNode); + let root = Nodes.setLeaf(nodes, 0, firstNode); this.root = Field(root); this.length = Field(1); @@ -438,7 +438,7 @@ class IndexedMerkleMapBase { // update internal hash nodes let i = Number(leaf.index.toBigInt()); - Nodes.setLeafNode(nodes, i, Leaf.hashNode(leaf).toBigInt()); + Nodes.setLeaf(nodes, i, Leaf.hashNode(leaf).toBigInt()); // update sorted list let leafValue = Leaf.toBigints(leaf); @@ -460,7 +460,7 @@ namespace Nodes { /** * Sets the leaf node at the given index, updates all parent nodes and returns the new root. */ - export function setLeafNode(nodes: Nodes, index: number, leaf: bigint) { + export function setLeaf(nodes: Nodes, index: number, leaf: bigint) { nodes[0][index] = leaf; let height = nodes.length; @@ -580,8 +580,8 @@ class OptionField extends Option(Field) {} * `getValue()` returns the value at the given index. * * We return - * `lowIndex` := max { i in [0, length) | getValue(i) <= target } - * `foundValue` := whether `getValue(lowIndex) == target` + * - `lowIndex := max { i in [0, length) | getValue(i) <= target }` + * - `foundValue` := whether `getValue(lowIndex) == target` */ function bisectUnique( target: bigint, @@ -597,13 +597,10 @@ function bisectUnique( if (getValue(iHigh) < target) return { lowIndex: iHigh, foundValue: false }; // invariant: 0 <= iLow <= lowIndex <= iHigh < length - // since we are either returning or reducing (iHigh - iLow), we'll eventually terminate correctly - while (true) { - if (iHigh === iLow) { - return { lowIndex: iLow, foundValue: getValue(iLow) === target }; - } - // either iLow + 1 = iHigh = iMid, or iLow < iMid < iHigh - // in both cases, the range gets strictly smaller + // since we are decreasing (iHigh - iLow) in every iteration, we'll terminate + while (iHigh !== iLow) { + // we have iLow < iMid <= iHigh + // in both branches, the range gets strictly smaller let iMid = Math.ceil((iLow + iHigh) / 2); if (getValue(iMid) <= target) { // iMid is in the candidate set, and strictly larger than iLow @@ -615,4 +612,6 @@ function bisectUnique( iHigh = iMid - 1; } } + + return { lowIndex: iLow, foundValue: getValue(iLow) === target }; } From 6461e1c38ba2321d603b5fa97aa9d48e779d5f99 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 12:47:55 +0200 Subject: [PATCH 66/93] prove updates correctly --- src/lib/provable/merkle-tree-indexed.ts | 118 ++++++++++++++++++------ 1 file changed, 89 insertions(+), 29 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 23f4d6e8cd..3bc157e2ea 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -153,7 +153,7 @@ class IndexedMerkleMapBase { // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); - this._proveInclusion(low, 'Invalid low node (root)'); + let lowPath = this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); // if the key does exist, we have lowNode.nextKey == key, and this line fails @@ -163,10 +163,10 @@ class IndexedMerkleMapBase { // update low node let newLow = { ...low, nextKey: key, nextIndex: index }; - this.root = this._computeRoot(newLow.index, Leaf.hashNode(newLow)); + this.root = this._proveUpdate(newLow, lowPath); this._setLeafUnconstrained(true, newLow); - // create and append new leaf + // create new leaf to append let leaf = Leaf.nextAfter(newLow, { key, value, @@ -174,7 +174,9 @@ class IndexedMerkleMapBase { nextIndex: low.nextIndex, }); - this.root = this._computeRoot(indexBits, Leaf.hashNode(leaf)); + // prove empty slot in the tree, and insert our leaf + let path = this._proveEmpty(indexBits); + this.root = this._proveUpdate(leaf, path); this.length = this.length.add(1); this._setLeafUnconstrained(false, leaf); } @@ -190,14 +192,14 @@ class IndexedMerkleMapBase { // prove that the key exists by presenting a leaf that contains it let self = Provable.witness(Leaf, () => this._findLeaf(key).self); - this._proveInclusion(self, 'Key does not exist in the tree'); + let path = this._proveInclusion(self, 'Key does not exist in the tree'); self.key.assertEquals(key, 'Invalid leaf (key)'); // at this point, we know that we have a valid update; so we can mutate internal data // update leaf let newSelf = { ...self, value }; - this.root = this._computeRoot(self.index, Leaf.hashNode(newSelf)); + this.root = this._proveUpdate(newSelf, path); this._setLeafUnconstrained(true, newSelf); } @@ -215,7 +217,7 @@ class IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); - this._proveInclusion(low, 'Invalid low node (root)'); + let lowPath = this._proveInclusion(low, 'Invalid low node (root)'); low.key.assertLessThan(key, 'Invalid low node (key)'); key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); @@ -226,17 +228,22 @@ class IndexedMerkleMapBase { let index = Provable.if(keyExists, low.nextIndex, this.length); let indexBits = index.toBits(this.height - 1); - // prove inclusion of this leaf if it exists - this._proveInclusionIf(keyExists, self, 'Invalid leaf (root)'); - assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); - // at this point, we know that we have a valid update or insertion; so we can mutate internal data // update low node, or leave it as is let newLow = { ...low, nextKey: key, nextIndex: index }; - this.root = this._computeRoot(low.index, Leaf.hashNode(newLow)); + this.root = this._proveUpdate(newLow, lowPath); this._setLeafUnconstrained(true, newLow); + // prove inclusion of this leaf if it exists + let path = this._proveInclusionOrEmpty( + keyExists, + indexBits, + self, + 'Invalid leaf (root)' + ); + assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); + // update leaf, or append a new one let newLeaf = Leaf.nextAfter(newLow, { key, @@ -244,7 +251,7 @@ class IndexedMerkleMapBase { nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), nextIndex: Provable.if(keyExists, self.nextIndex, low.nextIndex), }); - this.root = this._computeRoot(indexBits, Leaf.hashNode(newLeaf)); + this.root = this._proveUpdate(newLeaf, path); this.length = Provable.if(keyExists, this.length, this.length.add(1)); this._setLeafUnconstrained(keyExists, newLeaf); } @@ -346,8 +353,10 @@ class IndexedMerkleMapBase { // TODO: here, we don't actually care about the index, // so we could add a mode where `computeRoot()` doesn't prove it let node = Leaf.hashNode(leaf); - let root = this._computeRoot(leaf.index, node); + let { root, path } = this._computeRoot(leaf.index, node); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); + + return path; } /** @@ -355,44 +364,94 @@ class IndexedMerkleMapBase { */ _proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { let node = Leaf.hashNode(leaf); - let root = this._computeRoot(leaf.index, node); + let { root } = this._computeRoot(leaf.index, node); assert( condition.implies(root.equals(this.root)), message ?? 'Leaf is not included in the tree' ); } + /** + * Helper method to prove inclusion of an empty leaf in the tree. + * + * This validates the path against the current root, so that we can use it to insert a new leaf. + */ + _proveEmpty(index: Field | Bool[]) { + let node = Field(0n); + let { root, path } = this._computeRoot(index, node); + root.assertEquals(this.root, 'Leaf is not empty'); + + return path; + } + + /** + * Helper method to conditionally prove inclusion of a leaf in the tree. + * + * If the condition is false, we prove that the tree contains an empty leaf instead. + */ + _proveInclusionOrEmpty( + condition: Bool, + index: Field | Bool[], + leaf: Leaf, + message?: string + ) { + let node = Provable.if(condition, Leaf.hashNode(leaf), Field(0n)); + let { root, path } = this._computeRoot(index, node); + root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); + + return path; + } + + /** + * Helper method to update the root against a previously validated path. + * + * Returns the new root. + */ + _proveUpdate(leaf: Leaf, path: { witness: Field[]; index: Bool[] }) { + let node = Leaf.hashNode(leaf); + let { root } = this._computeRoot(path.index, node, path.witness); + return root; + } + /** * Helper method to compute the root given a leaf node and its index. * * The index can be given as a `Field` or as an array of bits. */ - _computeRoot(index: Field | Bool[], node: Field) { + _computeRoot(index: Field | Bool[], node: Field, witness?: Field[]) { let indexBits = index instanceof Field ? index.toBits(this.height - 1) : index; - assert(indexBits.length === this.height - 1, `Invalid index size`); + // if the witness was not passed in, we create it here + let witness_ = + witness ?? + Provable.witnessFields(this.height - 1, () => { + let witness: bigint[] = []; + let index = Number(Field.fromBits(indexBits)); + let { nodes } = this.data.get(); + + for (let level = 0; level < this.height - 1; level++) { + let i = index % 2 === 0 ? index + 1 : index - 1; + let sibling = Nodes.getNode(nodes, level, i, false); + witness.push(sibling); + index >>= 1; + } + + return witness; + }); - let indexU = Unconstrained.witness(() => - Number(Field.fromBits(indexBits).toBigInt()) - ); + assert(indexBits.length === this.height - 1, 'Invalid index size'); + assert(witness_.length === this.height - 1, 'Invalid witness size'); for (let level = 0; level < this.height - 1; level++) { - // in every iteration, we witness a sibling and hash it to get the parent node let isRight = indexBits[level]; - let sibling = Provable.witness(Field, () => { - let i = indexU.get(); - indexU.set(i >> 1); - let isLeft = !isRight.toBoolean(); - let nodes = this.data.get().nodes; - return Nodes.getNode(nodes, level, isLeft ? i + 1 : i - 1, false); - }); + let sibling = witness_[level]; let [right, left] = conditionalSwap(isRight, node, sibling); node = Poseidon.hash([left, right]); } // now, `node` is the root of the tree - return node; + return { root: node, path: { witness: witness_, index: indexBits } }; } /** @@ -479,6 +538,7 @@ namespace Nodes { nodes: Nodes, level: number, index: number, + // whether the node is required to be non-empty nonEmpty: boolean ) { let node = nodes[level]?.[index]; From 92d35f7d71a9140d64b26378e516732a491d959e Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 13:19:09 +0200 Subject: [PATCH 67/93] avoid computing index bits in circuit when not needed --- src/lib/provable/merkle-tree-indexed.ts | 35 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 3bc157e2ea..fa1368b649 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -350,10 +350,10 @@ class IndexedMerkleMapBase { * Helper method to prove inclusion of a leaf in the tree. */ _proveInclusion(leaf: Leaf, message?: string) { - // TODO: here, we don't actually care about the index, - // so we could add a mode where `computeRoot()` doesn't prove it let node = Leaf.hashNode(leaf); - let { root, path } = this._computeRoot(leaf.index, node); + // here, we don't care at which index the leaf is included, so we pass it in as unconstrained + let index = Unconstrained.from(leaf.index); + let { root, path } = this._computeRoot(node, index); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); return path; @@ -364,7 +364,9 @@ class IndexedMerkleMapBase { */ _proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { let node = Leaf.hashNode(leaf); - let { root } = this._computeRoot(leaf.index, node); + // here, we don't care at which index the leaf is included, so we pass it in as unconstrained + let index = Unconstrained.from(leaf.index); + let { root } = this._computeRoot(node, index); assert( condition.implies(root.equals(this.root)), message ?? 'Leaf is not included in the tree' @@ -376,9 +378,9 @@ class IndexedMerkleMapBase { * * This validates the path against the current root, so that we can use it to insert a new leaf. */ - _proveEmpty(index: Field | Bool[]) { + _proveEmpty(index: Bool[]) { let node = Field(0n); - let { root, path } = this._computeRoot(index, node); + let { root, path } = this._computeRoot(node, index); root.assertEquals(this.root, 'Leaf is not empty'); return path; @@ -391,12 +393,12 @@ class IndexedMerkleMapBase { */ _proveInclusionOrEmpty( condition: Bool, - index: Field | Bool[], + index: Bool[], leaf: Leaf, message?: string ) { let node = Provable.if(condition, Leaf.hashNode(leaf), Field(0n)); - let { root, path } = this._computeRoot(index, node); + let { root, path } = this._computeRoot(node, index); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); return path; @@ -407,9 +409,9 @@ class IndexedMerkleMapBase { * * Returns the new root. */ - _proveUpdate(leaf: Leaf, path: { witness: Field[]; index: Bool[] }) { + _proveUpdate(leaf: Leaf, path: { index: Bool[]; witness: Field[] }) { let node = Leaf.hashNode(leaf); - let { root } = this._computeRoot(path.index, node, path.witness); + let { root } = this._computeRoot(node, path.index, path.witness); return root; } @@ -418,9 +420,18 @@ class IndexedMerkleMapBase { * * The index can be given as a `Field` or as an array of bits. */ - _computeRoot(index: Field | Bool[], node: Field, witness?: Field[]) { + _computeRoot( + node: Field, + index: Unconstrained | Bool[], + witness?: Field[] + ) { + // if the index was passed in as unconstrained, we witness its bits here let indexBits = - index instanceof Field ? index.toBits(this.height - 1) : index; + index instanceof Unconstrained + ? Provable.witness(Provable.Array(Bool, this.height - 1), () => + index.get().toBits(this.height - 1) + ) + : index; // if the witness was not passed in, we create it here let witness_ = From b0ea55d4ca27efdb682a615e7a99f33f57a5d189 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 13:39:13 +0200 Subject: [PATCH 68/93] remove nextIndex --- src/lib/provable/merkle-tree-indexed.ts | 27 +++++++------------ .../provable/test/merkle-tree.unit-test.ts | 1 - 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index fa1368b649..66a5e3772f 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -135,7 +135,6 @@ class IndexedMerkleMapBase { // maximum, which is always greater than any key that is a hash nextKey: Field.ORDER - 1n, index: 0n, - nextIndex: 0n, // TODO: ok? }; /** @@ -162,7 +161,7 @@ class IndexedMerkleMapBase { // at this point, we know that we have a valid insertion; so we can mutate internal data // update low node - let newLow = { ...low, nextKey: key, nextIndex: index }; + let newLow = { ...low, nextKey: key }; this.root = this._proveUpdate(newLow, lowPath); this._setLeafUnconstrained(true, newLow); @@ -170,8 +169,8 @@ class IndexedMerkleMapBase { let leaf = Leaf.nextAfter(newLow, { key, value, + index, nextKey: low.nextKey, - nextIndex: low.nextIndex, }); // prove empty slot in the tree, and insert our leaf @@ -225,13 +224,13 @@ class IndexedMerkleMapBase { let keyExists = low.nextKey.equals(key); // the leaf's index depends on whether it exists - let index = Provable.if(keyExists, low.nextIndex, this.length); + let index = Provable.if(keyExists, self.index, this.length); let indexBits = index.toBits(this.height - 1); // at this point, we know that we have a valid update or insertion; so we can mutate internal data // update low node, or leave it as is - let newLow = { ...low, nextKey: key, nextIndex: index }; + let newLow = { ...low, nextKey: key }; this.root = this._proveUpdate(newLow, lowPath); this._setLeafUnconstrained(true, newLow); @@ -248,8 +247,8 @@ class IndexedMerkleMapBase { let newLeaf = Leaf.nextAfter(newLow, { key, value, + index, nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), - nextIndex: Provable.if(keyExists, self.nextIndex, low.nextIndex), }); this.root = this._proveUpdate(newLeaf, path); this.length = Provable.if(keyExists, this.length, this.length.add(1)); @@ -394,7 +393,7 @@ class IndexedMerkleMapBase { _proveInclusionOrEmpty( condition: Bool, index: Bool[], - leaf: Leaf, + leaf: BaseLeaf, message?: string ) { let node = Provable.if(condition, Leaf.hashNode(leaf), Field(0n)); @@ -409,7 +408,7 @@ class IndexedMerkleMapBase { * * Returns the new root. */ - _proveUpdate(leaf: Leaf, path: { index: Bool[]; witness: Field[] }) { + _proveUpdate(leaf: BaseLeaf, path: { index: Bool[]; witness: Field[] }) { let node = Leaf.hashNode(leaf); let { root } = this._computeRoot(node, path.index, path.witness); return root; @@ -581,7 +580,6 @@ class BaseLeaf extends Struct({ key: Field, value: Field, nextKey: Field, - nextIndex: Field, }) {} class Leaf extends Struct({ @@ -591,8 +589,6 @@ class Leaf extends Struct({ nextKey: Field, index: Field, - nextIndex: Field, - sortedIndex: Unconstrained.provableWithEmpty(0), }) { /** @@ -607,13 +603,12 @@ class Leaf extends Struct({ /** * Create a new leaf, given its low node. */ - static nextAfter(low: Leaf, leaf: BaseLeaf): Leaf { + static nextAfter(low: Leaf, leaf: BaseLeaf & { index: Field }): Leaf { return { key: leaf.key, value: leaf.value, nextKey: leaf.nextKey, - nextIndex: leaf.nextIndex, - index: low.nextIndex, + index: leaf.index, sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), }; } @@ -624,19 +619,15 @@ class Leaf extends Struct({ value: leaf.value.toBigInt(), nextKey: leaf.nextKey.toBigInt(), index: leaf.index.toBigInt(), - nextIndex: leaf.nextIndex.toBigInt(), }; } } type LeafValue = { value: bigint; - key: bigint; nextKey: bigint; - index: bigint; - nextIndex: bigint; }; class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index add6f9e149..8893efeac9 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -178,7 +178,6 @@ console.log( value: sorted[i].value, nextKey: sorted[i + 1]?.key ?? Field.ORDER - 1n, index: sorted[i].index, - nextIndex: sorted[i + 1]?.index ?? 0n, }); } } From c189fab52ad1f7a3112ea1129817141688b610f2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 14:20:01 +0200 Subject: [PATCH 69/93] safer/more logical representation of leaf index --- src/lib/provable/merkle-tree-indexed.ts | 50 ++++++++++--------- .../provable/test/merkle-tree.unit-test.ts | 8 +-- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 66a5e3772f..a88d66957f 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -134,7 +134,7 @@ class IndexedMerkleMapBase { value: 0n, // maximum, which is always greater than any key that is a hash nextKey: Field.ORDER - 1n, - index: 0n, + index: 0, }; /** @@ -166,10 +166,9 @@ class IndexedMerkleMapBase { this._setLeafUnconstrained(true, newLow); // create new leaf to append - let leaf = Leaf.nextAfter(newLow, { + let leaf = Leaf.nextAfter(newLow, index, { key, value, - index, nextKey: low.nextKey, }); @@ -224,7 +223,8 @@ class IndexedMerkleMapBase { let keyExists = low.nextKey.equals(key); // the leaf's index depends on whether it exists - let index = Provable.if(keyExists, self.index, this.length); + let index = Provable.witness(Field, () => self.index.get()); + index = Provable.if(keyExists, index, this.length); let indexBits = index.toBits(this.height - 1); // at this point, we know that we have a valid update or insertion; so we can mutate internal data @@ -244,10 +244,9 @@ class IndexedMerkleMapBase { assert(keyExists.implies(self.key.equals(key)), 'Invalid leaf (key)'); // update leaf, or append a new one - let newLeaf = Leaf.nextAfter(newLow, { + let newLeaf = Leaf.nextAfter(newLow, index, { key, value, - index, nextKey: Provable.if(keyExists, self.nextKey, low.nextKey), }); this.root = this._proveUpdate(newLeaf, path); @@ -351,8 +350,7 @@ class IndexedMerkleMapBase { _proveInclusion(leaf: Leaf, message?: string) { let node = Leaf.hashNode(leaf); // here, we don't care at which index the leaf is included, so we pass it in as unconstrained - let index = Unconstrained.from(leaf.index); - let { root, path } = this._computeRoot(node, index); + let { root, path } = this._computeRoot(node, leaf.index); root.assertEquals(this.root, message ?? 'Leaf is not included in the tree'); return path; @@ -364,8 +362,7 @@ class IndexedMerkleMapBase { _proveInclusionIf(condition: Bool, leaf: Leaf, message?: string) { let node = Leaf.hashNode(leaf); // here, we don't care at which index the leaf is included, so we pass it in as unconstrained - let index = Unconstrained.from(leaf.index); - let { root } = this._computeRoot(node, index); + let { root } = this._computeRoot(node, leaf.index); assert( condition.implies(root.equals(this.root)), message ?? 'Leaf is not included in the tree' @@ -421,14 +418,14 @@ class IndexedMerkleMapBase { */ _computeRoot( node: Field, - index: Unconstrained | Bool[], + index: Unconstrained | Bool[], witness?: Field[] ) { // if the index was passed in as unconstrained, we witness its bits here let indexBits = index instanceof Unconstrained ? Provable.witness(Provable.Array(Bool, this.height - 1), () => - index.get().toBits(this.height - 1) + Field(index.get()).toBits(this.height - 1) ) : index; @@ -481,7 +478,7 @@ class IndexedMerkleMapBase { if (key === 0n) return { low: Leaf.toValue(Leaf.empty()), - self: { ...leaves[0], sortedIndex: Unconstrained.from(0) }, + self: Leaf.fromBigints(leaves[0], 0), }; let { lowIndex, foundValue } = bisectUnique( @@ -490,11 +487,11 @@ class IndexedMerkleMapBase { leaves.length ); let iLow = foundValue ? lowIndex - 1 : lowIndex; - let low = { ...leaves[iLow], sortedIndex: Unconstrained.from(iLow) }; + let low = Leaf.fromBigints(leaves[iLow], iLow); let iSelf = foundValue ? lowIndex : 0; let selfBase = foundValue ? leaves[lowIndex] : Leaf.toBigints(Leaf.empty()); - let self = { ...selfBase, sortedIndex: Unconstrained.from(iSelf) }; + let self = Leaf.fromBigints(selfBase, iSelf); return { low, self }; } @@ -506,7 +503,7 @@ class IndexedMerkleMapBase { let { nodes, sortedLeaves } = this.data.get(); // update internal hash nodes - let i = Number(leaf.index.toBigInt()); + let i = leaf.index.get(); Nodes.setLeaf(nodes, i, Leaf.hashNode(leaf).toBigInt()); // update sorted list @@ -584,11 +581,10 @@ class BaseLeaf extends Struct({ class Leaf extends Struct({ value: Field, - key: Field, nextKey: Field, - index: Field, + index: Unconstrained.provableWithEmpty(0), sortedIndex: Unconstrained.provableWithEmpty(0), }) { /** @@ -601,14 +597,14 @@ class Leaf extends Struct({ } /** - * Create a new leaf, given its low node. + * Create a new leaf, given its low node and index. */ - static nextAfter(low: Leaf, leaf: BaseLeaf & { index: Field }): Leaf { + static nextAfter(low: Leaf, index: Field, leaf: BaseLeaf): Leaf { return { key: leaf.key, value: leaf.value, nextKey: leaf.nextKey, - index: leaf.index, + index: Unconstrained.witness(() => Number(index)), sortedIndex: Unconstrained.witness(() => low.sortedIndex.get() + 1), }; } @@ -618,7 +614,15 @@ class Leaf extends Struct({ key: leaf.key.toBigInt(), value: leaf.value.toBigInt(), nextKey: leaf.nextKey.toBigInt(), - index: leaf.index.toBigInt(), + index: leaf.index.get(), + }; + } + + static fromBigints(leaf: LeafValue, sortedIndex: number) { + return { + ...leaf, + index: Unconstrained.from(leaf.index), + sortedIndex: Unconstrained.from(sortedIndex), }; } } @@ -627,7 +631,7 @@ type LeafValue = { value: bigint; key: bigint; nextKey: bigint; - index: bigint; + index: number; }; class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 8893efeac9..3d8410c3fd 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -165,10 +165,10 @@ console.log( // data sorted by key: let sorted = [ - { key: 0n, value: 12n, index: 0n }, - { key: 1n, value: 17n, index: 2n }, - { key: 2n, value: 15n, index: 1n }, - { key: 4n, value: 16n, index: 3n }, + { key: 0n, value: 12n, index: 0 }, + { key: 1n, value: 17n, index: 2 }, + { key: 2n, value: 15n, index: 1 }, + { key: 4n, value: 16n, index: 3 }, ]; let sortedLeaves = map.data.get().sortedLeaves; From 3d8c78474427fe5a8b8d35576f4d9fce75a39b76 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 14:26:41 +0200 Subject: [PATCH 70/93] misc improvements --- src/lib/provable/merkle-tree-indexed.ts | 38 ++++++++++++------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index a88d66957f..d2dc4a6a79 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -58,11 +58,6 @@ function IndexedMerkleMap(height: number): typeof IndexedMerkleMapBase { ); return class IndexedMerkleMap extends IndexedMerkleMapBase { - constructor() { - // we can't access the abstract `height` property in the base constructor - super(); - } - get height() { return height; } @@ -76,7 +71,7 @@ const provableBase = { length: Field, data: Unconstrained.provableWithEmpty({ nodes: [] as (bigint | undefined)[][], - sortedLeaves: [] as LeafValue[], + sortedLeaves: [] as StoredLeaf[], }), }; @@ -86,11 +81,11 @@ class IndexedMerkleMapBase { length: Field; // length of the leaves array // static data defining constraints - get height() { - return 0; + get height(): number { + throw Error('Height must be defined in a subclass'); } - // the raw data stored in the tree, plus helper structures + // the raw data stored in the tree readonly data: Unconstrained<{ // for every level, an array of hashes readonly nodes: (bigint | undefined)[][]; @@ -100,7 +95,7 @@ class IndexedMerkleMapBase { // sortedLeaves[0].key = 0 // sortedLeaves[n-1].nextKey = Field.ORDER - 1 // for i=0,...n-2, sortedLeaves[i].nextKey = sortedLeaves[i+1].key - readonly sortedLeaves: LeafValue[]; + readonly sortedLeaves: StoredLeaf[]; }>; // we'd like to do `abstract static provable` here but that's not supported @@ -110,7 +105,7 @@ class IndexedMerkleMapBase { > = undefined as any; /** - * Creates a new, empty Indexed Merkle Map, given its height. + * Creates a new, empty Indexed Merkle Map. */ constructor() { let height = this.height; @@ -478,7 +473,7 @@ class IndexedMerkleMapBase { if (key === 0n) return { low: Leaf.toValue(Leaf.empty()), - self: Leaf.fromBigints(leaves[0], 0), + self: Leaf.fromStored(leaves[0], 0), }; let { lowIndex, foundValue } = bisectUnique( @@ -487,11 +482,11 @@ class IndexedMerkleMapBase { leaves.length ); let iLow = foundValue ? lowIndex - 1 : lowIndex; - let low = Leaf.fromBigints(leaves[iLow], iLow); + let low = Leaf.fromStored(leaves[iLow], iLow); let iSelf = foundValue ? lowIndex : 0; - let selfBase = foundValue ? leaves[lowIndex] : Leaf.toBigints(Leaf.empty()); - let self = Leaf.fromBigints(selfBase, iSelf); + let selfBase = foundValue ? leaves[lowIndex] : Leaf.toStored(Leaf.empty()); + let self = Leaf.fromStored(selfBase, iSelf); return { low, self }; } @@ -507,7 +502,7 @@ class IndexedMerkleMapBase { Nodes.setLeaf(nodes, i, Leaf.hashNode(leaf).toBigInt()); // update sorted list - let leafValue = Leaf.toBigints(leaf); + let leafValue = Leaf.toStored(leaf); let iSorted = leaf.sortedIndex.get(); if (Bool(leafExists).toBoolean()) { @@ -584,6 +579,7 @@ class Leaf extends Struct({ key: Field, nextKey: Field, + // auxiliary data that tells us where the leaf is stored index: Unconstrained.provableWithEmpty(0), sortedIndex: Unconstrained.provableWithEmpty(0), }) { @@ -591,7 +587,7 @@ class Leaf extends Struct({ * Compute a leaf node: the hash of a leaf that becomes part of the Merkle tree. */ static hashNode(leaf: From) { - // note: we don't have to include the `index` in the leaf hash, + // note: we don't have to include the index in the leaf hash, // because computing the root already commits to the index return Poseidon.hashPacked(BaseLeaf, BaseLeaf.fromValue(leaf)); } @@ -609,7 +605,9 @@ class Leaf extends Struct({ }; } - static toBigints(leaf: Leaf): LeafValue { + // convert to/from internally stored format + + static toStored(leaf: Leaf): StoredLeaf { return { key: leaf.key.toBigInt(), value: leaf.value.toBigInt(), @@ -618,7 +616,7 @@ class Leaf extends Struct({ }; } - static fromBigints(leaf: LeafValue, sortedIndex: number) { + static fromStored(leaf: StoredLeaf, sortedIndex: number) { return { ...leaf, index: Unconstrained.from(leaf.index), @@ -627,7 +625,7 @@ class Leaf extends Struct({ } } -type LeafValue = { +type StoredLeaf = { value: bigint; key: bigint; nextKey: bigint; From c6445ab456c6fc70e56d6f6665e1d90ea9ca6ca6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 14:29:19 +0200 Subject: [PATCH 71/93] unconstrained tweak --- src/lib/provable/types/unconstrained.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/types/unconstrained.ts b/src/lib/provable/types/unconstrained.ts index dbdba166f7..857a1d9665 100644 --- a/src/lib/provable/types/unconstrained.ts +++ b/src/lib/provable/types/unconstrained.ts @@ -86,7 +86,8 @@ and Provable.asProver() blocks, which execute outside the proof. * let xWrapped = Unconstrained.witness(() => Provable.toConstant(type, x)); * ``` */ - static from(value: T) { + static from(value: T | Unconstrained) { + if (value instanceof Unconstrained) return value; return new Unconstrained(true, value); } From 25ddcceed1bcd62da54895d2b20ff67730349afd Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 22:15:46 +0200 Subject: [PATCH 72/93] make indexed map clonable --- src/lib/provable/merkle-tree-indexed.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index d2dc4a6a79..c77c7b6480 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -98,6 +98,19 @@ class IndexedMerkleMapBase { readonly sortedLeaves: StoredLeaf[]; }>; + clone() { + let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); + cloned.root = this.root; + cloned.length = this.length; + cloned.data.updateAsProver(({ nodes, sortedLeaves }) => { + return { + nodes: nodes.map((row) => [...row]), + sortedLeaves: [...sortedLeaves], + }; + }); + return cloned; + } + // we'd like to do `abstract static provable` here but that's not supported static provable: Provable< IndexedMerkleMapBase, @@ -626,10 +639,10 @@ class Leaf extends Struct({ } type StoredLeaf = { - value: bigint; - key: bigint; - nextKey: bigint; - index: number; + readonly value: bigint; + readonly key: bigint; + readonly nextKey: bigint; + readonly index: number; }; class LeafPair extends Struct({ low: Leaf, self: Leaf }) {} From d59e84b388abfe66fafddfbad03407390002918b Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 29 May 2024 22:27:44 +0200 Subject: [PATCH 73/93] return the previous value --- src/lib/provable/merkle-tree-indexed.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index c77c7b6480..be62e250e2 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -191,8 +191,10 @@ class IndexedMerkleMapBase { * Update an existing leaf `(key, value)`. * * Proves that the `key` exists. + * + * Returns the previous value. */ - update(key: Field | bigint, value: Field | bigint) { + update(key: Field | bigint, value: Field | bigint): Field { key = Field(key); value = Field(value); @@ -207,6 +209,8 @@ class IndexedMerkleMapBase { let newSelf = { ...self, value }; this.root = this._proveUpdate(newSelf, path); this._setLeafUnconstrained(true, newSelf); + + return self.value; } /** @@ -216,8 +220,10 @@ class IndexedMerkleMapBase { * can use it if you don't know whether the key exists or not. * * However, this comes at an efficiency cost, so prefer to use `insert()` or `update()` if you know whether the key exists. + * + * Returns the previous value, as an option (which is `None` if the key didn't exist before). */ - set(key: Field | bigint, value: Field | bigint) { + set(key: Field | bigint, value: Field | bigint): Option { key = Field(key); value = Field(value); @@ -260,6 +266,9 @@ class IndexedMerkleMapBase { this.root = this._proveUpdate(newLeaf, path); this.length = Provable.if(keyExists, this.length, this.length.add(1)); this._setLeafUnconstrained(keyExists, newLeaf); + + // return the previous value + return new OptionField({ isSome: keyExists, value: self.value }); } /** From 4c8b3e6cccee0053abbb3d19190b03bf9cf68c92 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 18:17:55 +0200 Subject: [PATCH 74/93] try to make 0 the max to support all keys, for easy dummy checks --- src/lib/provable/field.ts | 2 +- src/lib/provable/merkle-tree-indexed.ts | 83 +++++++++++++++---- .../provable/test/merkle-tree.unit-test.ts | 16 ++-- 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/src/lib/provable/field.ts b/src/lib/provable/field.ts index 4c5009146f..f086811231 100644 --- a/src/lib/provable/field.ts +++ b/src/lib/provable/field.ts @@ -654,7 +654,7 @@ class Field { * Assert that this {@link Field} is less than another "field-like" value. * * Note: This uses fewer constraints than `x.lessThan(y).assertTrue()`. - * See {@link Field.lessThan} for more details. + * See {@link lessThan} for more details. * * **Important**: If an assertion fails, the code throws an error. * diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index be62e250e2..4396c8b1a6 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -141,7 +141,7 @@ class IndexedMerkleMapBase { key: 0n, value: 0n, // maximum, which is always greater than any key that is a hash - nextKey: Field.ORDER - 1n, + nextKey: 0n, index: 0, }; @@ -161,10 +161,12 @@ class IndexedMerkleMapBase { // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); let lowPath = this._proveInclusion(low, 'Invalid low node (root)'); - low.key.assertLessThan(key, 'Invalid low node (key)'); - - // if the key does exist, we have lowNode.nextKey == key, and this line fails - key.assertLessThan(low.nextKey, 'Key already exists in the tree'); + assertInRangeStrict( + low.key, + key, + low.nextKey, + 'Key already exists in the tree' + ); // at this point, we know that we have a valid insertion; so we can mutate internal data @@ -230,8 +232,7 @@ class IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); let lowPath = this._proveInclusion(low, 'Invalid low node (root)'); - low.key.assertLessThan(key, 'Invalid low node (key)'); - key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); + assertInRange(low.key, key, low.nextKey, 'Invalid low node (key)'); // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); @@ -301,8 +302,7 @@ class IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); this._proveInclusion(low, 'Invalid low node (root)'); - low.key.assertLessThan(key, 'Invalid low node (key)'); - key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); + assertInRange(low.key, key, low.nextKey, 'Invalid low node (key)'); // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); @@ -337,8 +337,9 @@ class IndexedMerkleMapBase { // prove that the key does not exist yet, by showing a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); this._proveInclusion(low, 'Invalid low node (root)'); - low.key.assertLessThan(key, 'Invalid low node (key)'); - key.assertLessThan( + assertInRangeStrict( + low.key, + key, low.nextKey, message ?? 'Key already exists in the tree' ); @@ -353,8 +354,7 @@ class IndexedMerkleMapBase { // prove that the key does not exist yet, by showing a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); this._proveInclusion(low, 'Invalid low node (root)'); - low.key.assertLessThan(key, 'Invalid low node (key)'); - key.assertLessThanOrEqual(low.nextKey, 'Invalid low node (next key)'); + assertInRange(low.key, key, low.nextKey, 'Invalid low node (key)'); return low.nextKey.equals(key); } @@ -494,7 +494,7 @@ class IndexedMerkleMapBase { // and reject it using comparison constraints if (key === 0n) return { - low: Leaf.toValue(Leaf.empty()), + low: Leaf.fromStored(leaves[leaves.length - 1], leaves.length - 1), self: Leaf.fromStored(leaves[0], 0), }; @@ -701,3 +701,58 @@ function bisectUnique( return { lowIndex: iLow, foundValue: getValue(iLow) === target }; } + +/** + * Assert that x in (low, high), i.e. low < x < high, with the following exceptions: + * + * - high=0 is treated as the maximum value, so x in (low, 0) always succeeds if only low < x; except for x = 0. + * - x=0 is also treated as the maximum value, so 0 in (low, high) always fails, because x >= high. + */ +function assertInRangeStrict( + low: Field, + x: Field, + high: Field, + message?: string +) { + Provable.log('assertInRangeStrict', { low, x, high }); + // exclude x=0 + x.assertNotEquals(0n, message ?? '0 is not in any range'); + + // normal assertion for low < x + low.assertLessThan(x, message ?? 'Expected low < x'); + + // for x < high, use a safe comparison that also works if high=0 + let highIsZero = high.equals(0n); + let xSafe = Provable.witness(Field, () => (highIsZero.toBoolean() ? 0n : x)); + let highSafe = Provable.witness(Field, () => + highIsZero.toBoolean() ? 1n : high + ); + xSafe.assertLessThan(highSafe, message); + assert(xSafe.equals(x).or(highIsZero), message); + assert(highSafe.equals(high).or(highIsZero), message); +} + +/** + * Assert that x in (low, high], i.e. low < x <= high, with the following exceptions: + * + * - high=0 is treated as the maximum value, so x in (low, 0] always succeeds if only low < x. + * - x=0 is also treated as the maximum value, so 0 in (low, high] fails except if high=0. + * - note: 0 in (n, 0] succeeds for any n! + */ +function assertInRange(low: Field, x: Field, high: Field, message?: string) { + Provable.log('assertInRange', { low, x, high }); + + // for low < x, we need to handle the x=0 case separately + let xIsZero = x.equals(0n); + let lowSafe = Provable.witness(Field, () => (xIsZero.toBoolean() ? 0n : low)); + let xSafe1 = Provable.witness(Field, () => (xIsZero.toBoolean() ? 1n : x)); + lowSafe.assertLessThan(xSafe1, message); + assert(lowSafe.equals(low).or(xIsZero), message); + assert(xSafe1.equals(x).or(xIsZero), message); + + // for x <= high, we need to handle the high=0 case separately + let highIsZero = high.equals(0n); + let xSafe0 = Provable.witness(Field, () => (highIsZero.toBoolean() ? 0n : x)); + xSafe0.assertLessThanOrEqual(high, message); + assert(xSafe0.equals(x).or(highIsZero), message); +} diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 3d8410c3fd..2e4ccf5b68 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -112,11 +112,13 @@ console.log( map.insert(2n, 14n); map.insert(1n, 13n); + // map.insert(-1n, 11n); + // map.set(-1n, 12n); - // -1 (the max value) can't be inserted, because there's always a value pointing to it, - // and yet it's not included as a leaf - expect(() => map.insert(-1n, 11n)).toThrow('Key already exists'); - expect(() => map.set(-1n, 12n)).toThrow('Invalid leaf'); + // // -1 (the max value) can't be inserted, because there's always a value pointing to it, + // // and yet it's not included as a leaf + // expect(() => map.insert(-1n, 11n)).toThrow('Key already exists'); + // expect(() => map.set(-1n, 12n)).toThrow('Invalid leaf'); expect(map.getOption(1n).assertSome().toBigInt()).toEqual(13n); expect(map.getOption(2n).assertSome().toBigInt()).toEqual(14n); @@ -127,7 +129,11 @@ console.log( expect(map.getOption(2n).assertSome().toBigInt()).toEqual(15n); // TODO get() doesn't work on 0n because the low node checks fail - // expect(map.get(0n).assertSome().toBigInt()).toEqual(12n); + expect(map.get(0n).toBigInt()).toEqual(12n); + expect(map.getOption(0n).assertSome().toBigInt()).toEqual(12n); + + // TODO set() doesn't work on 0n because the low node checks fail + map.set(0n, 12n); // can't insert the same key twice expect(() => map.insert(1n, 17n)).toThrow('Key already exists'); From 225c2548736e3cf6b08ee5a05147e85cff4304a3 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Mon, 3 Jun 2024 10:03:57 -0700 Subject: [PATCH 75/93] fix(run-berkeley.ts): update mina and archive URLs to point to new API endpoints for better reliability and performance --- src/examples/zkapps/voting/run-berkeley.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/examples/zkapps/voting/run-berkeley.ts b/src/examples/zkapps/voting/run-berkeley.ts index 38ed59b2b2..5255de76d2 100644 --- a/src/examples/zkapps/voting/run-berkeley.ts +++ b/src/examples/zkapps/voting/run-berkeley.ts @@ -20,8 +20,8 @@ import { import { getResults, vote } from './voting-lib.js'; const Berkeley = Mina.Network({ - mina: 'https://proxy.berkeley.minaexplorer.com/graphql', - archive: 'https://archive-node-api.p42.xyz/', + mina: 'https://api.minascan.io/node/devnet/v1/graphql', + archive: 'https://api.minascan.io/archive/devnet/v1/graphql', }); Mina.setActiveInstance(Berkeley); From dd7a41abbbbbca05327dfe5dff48c275857de50b Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 21:10:00 +0200 Subject: [PATCH 76/93] fix case where sorted index overflows --- src/lib/provable/merkle-tree-indexed.ts | 7 +++-- .../provable/test/merkle-tree.unit-test.ts | 27 +++++++------------ 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 4396c8b1a6..273f1231a1 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -528,7 +528,9 @@ class IndexedMerkleMapBase { let iSorted = leaf.sortedIndex.get(); if (Bool(leafExists).toBoolean()) { - sortedLeaves[iSorted] = leafValue; + // for key=0, the sorted index overflows the length because we compute it as low.sortedIndex + 1 + // in that case, it should wrap back to 0 + sortedLeaves[iSorted % sortedLeaves.length] = leafValue; } else { sortedLeaves.splice(iSorted, 0, leafValue); } @@ -714,7 +716,6 @@ function assertInRangeStrict( high: Field, message?: string ) { - Provable.log('assertInRangeStrict', { low, x, high }); // exclude x=0 x.assertNotEquals(0n, message ?? '0 is not in any range'); @@ -740,8 +741,6 @@ function assertInRangeStrict( * - note: 0 in (n, 0] succeeds for any n! */ function assertInRange(low: Field, x: Field, high: Field, message?: string) { - Provable.log('assertInRange', { low, x, high }); - // for low < x, we need to handle the x=0 case separately let xIsZero = x.equals(0n); let lowSafe = Provable.witness(Field, () => (xIsZero.toBoolean() ? 0n : low)); diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 2e4ccf5b68..515a02ad42 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -102,6 +102,7 @@ console.log( // some manual tests for IndexedMerkleMap { let map = new (IndexedMerkleMap(3))(); + const minus1 = Field.ORDER - 1n; // there's 1 element in the map at the beginning // check initial root against `MerkleTree` implementation @@ -110,31 +111,20 @@ console.log( initialTree.setLeaf(0n, Leaf.hashNode(IndexedMerkleMap(3)._firstLeaf)); expect(map.root).toEqual(initialTree.getRoot()); - map.insert(2n, 14n); + map.insert(-1n, 14n); // -1 is the largest possible value map.insert(1n, 13n); - // map.insert(-1n, 11n); - // map.set(-1n, 12n); - - // // -1 (the max value) can't be inserted, because there's always a value pointing to it, - // // and yet it's not included as a leaf - // expect(() => map.insert(-1n, 11n)).toThrow('Key already exists'); - // expect(() => map.set(-1n, 12n)).toThrow('Invalid leaf'); expect(map.getOption(1n).assertSome().toBigInt()).toEqual(13n); - expect(map.getOption(2n).assertSome().toBigInt()).toEqual(14n); + expect(map.getOption(-1n).assertSome().toBigInt()).toEqual(14n); expect(map.getOption(3n).isSome.toBoolean()).toEqual(false); - map.update(2n, 15n); + map.update(-1n, 15n); map.update(0n, 12n); - expect(map.getOption(2n).assertSome().toBigInt()).toEqual(15n); + expect(map.getOption(-1n).assertSome().toBigInt()).toEqual(15n); - // TODO get() doesn't work on 0n because the low node checks fail expect(map.get(0n).toBigInt()).toEqual(12n); expect(map.getOption(0n).assertSome().toBigInt()).toEqual(12n); - // TODO set() doesn't work on 0n because the low node checks fail - map.set(0n, 12n); - // can't insert the same key twice expect(() => map.insert(1n, 17n)).toThrow('Key already exists'); @@ -143,6 +133,7 @@ console.log( map.set(4n, 16n); map.set(1n, 17n); + map.set(0n, 12n); expect(map.get(4n).toBigInt()).toEqual(16n); expect(map.getOption(1n).assertSome().toBigInt()).toEqual(17n); expect(map.getOption(5n).isSome.toBoolean()).toEqual(false); @@ -155,7 +146,7 @@ console.log( expect(map.length.toBigInt()).toEqual(4n); // check that internal nodes exactly match `MerkleTree` implementation - let keys = [0n, 2n, 1n, 4n]; // insertion order + let keys = [0n, minus1, 1n, 4n]; // insertion order let leafNodes = keys.map((key) => Leaf.hashNode(map._findLeaf(key).self)); let tree = new MerkleTree(3); tree.fill(leafNodes); @@ -173,8 +164,8 @@ console.log( let sorted = [ { key: 0n, value: 12n, index: 0 }, { key: 1n, value: 17n, index: 2 }, - { key: 2n, value: 15n, index: 1 }, { key: 4n, value: 16n, index: 3 }, + { key: minus1, value: 15n, index: 1 }, ]; let sortedLeaves = map.data.get().sortedLeaves; @@ -182,7 +173,7 @@ console.log( expect(sortedLeaves[i]).toEqual({ key: sorted[i].key, value: sorted[i].value, - nextKey: sorted[i + 1]?.key ?? Field.ORDER - 1n, + nextKey: sorted[i + 1]?.key ?? 0n, index: sorted[i].index, }); } From c1a021e05b308352f3fa2736e21bccf047a32f31 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 21:42:16 +0200 Subject: [PATCH 77/93] more comments --- src/lib/provable/merkle-tree-indexed.ts | 38 ++++++++++++++----------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 273f1231a1..a1ed19103b 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -140,7 +140,8 @@ class IndexedMerkleMapBase { static _firstLeaf = { key: 0n, value: 0n, - // maximum, which is always greater than any key that is a hash + // the 0 key encodes the minimum and maximum at the same time + // so, if a second node is inserted, it will get `nextKey = 0`, and thus point to the first node nextKey: 0n, index: 0, }; @@ -161,7 +162,8 @@ class IndexedMerkleMapBase { // prove that the key doesn't exist yet by presenting a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); let lowPath = this._proveInclusion(low, 'Invalid low node (root)'); - assertInRangeStrict( + // if the key does exist, we have lowNode.nextKey == key, and this line fails + assertStrictlyBetween( low.key, key, low.nextKey, @@ -232,7 +234,7 @@ class IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); let lowPath = this._proveInclusion(low, 'Invalid low node (root)'); - assertInRange(low.key, key, low.nextKey, 'Invalid low node (key)'); + assertBetween(low.key, key, low.nextKey, 'Invalid low node (key)'); // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); @@ -302,7 +304,7 @@ class IndexedMerkleMapBase { // prove whether the key exists or not, by showing a valid low node let { low, self } = Provable.witness(LeafPair, () => this._findLeaf(key)); this._proveInclusion(low, 'Invalid low node (root)'); - assertInRange(low.key, key, low.nextKey, 'Invalid low node (key)'); + assertBetween(low.key, key, low.nextKey, 'Invalid low node (key)'); // the key exists iff lowNode.nextKey == key let keyExists = low.nextKey.equals(key); @@ -337,7 +339,7 @@ class IndexedMerkleMapBase { // prove that the key does not exist yet, by showing a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); this._proveInclusion(low, 'Invalid low node (root)'); - assertInRangeStrict( + assertStrictlyBetween( low.key, key, low.nextKey, @@ -354,7 +356,7 @@ class IndexedMerkleMapBase { // prove that the key does not exist yet, by showing a valid low node let low = Provable.witness(Leaf, () => this._findLeaf(key).low); this._proveInclusion(low, 'Invalid low node (root)'); - assertInRange(low.key, key, low.nextKey, 'Invalid low node (key)'); + assertBetween(low.key, key, low.nextKey, 'Invalid low node (key)'); return low.nextKey.equals(key); } @@ -704,23 +706,25 @@ function bisectUnique( return { lowIndex: iLow, foundValue: getValue(iLow) === target }; } +// custom comparison methods where 0 can act as the min and max value simultaneously + /** - * Assert that x in (low, high), i.e. low < x < high, with the following exceptions: + * Assert that `x in (low, high)`, i.e. low < x < high, with the following exceptions: * - * - high=0 is treated as the maximum value, so x in (low, 0) always succeeds if only low < x; except for x = 0. - * - x=0 is also treated as the maximum value, so 0 in (low, high) always fails, because x >= high. + * - high=0 is treated as the maximum value, so `x in (low, 0)` always succeeds if only low < x; except for x = 0. + * - x=0 is also treated as the maximum value, so `0 in (low, high)` always fails, because x >= high. */ -function assertInRangeStrict( +function assertStrictlyBetween( low: Field, x: Field, high: Field, message?: string ) { // exclude x=0 - x.assertNotEquals(0n, message ?? '0 is not in any range'); + x.assertNotEquals(0n, message ?? '0 is not in any strict range'); // normal assertion for low < x - low.assertLessThan(x, message ?? 'Expected low < x'); + low.assertLessThan(x, message); // for x < high, use a safe comparison that also works if high=0 let highIsZero = high.equals(0n); @@ -734,13 +738,13 @@ function assertInRangeStrict( } /** - * Assert that x in (low, high], i.e. low < x <= high, with the following exceptions: + * Assert that `x in (low, high]`, i.e. low < x <= high, with the following exceptions: * - * - high=0 is treated as the maximum value, so x in (low, 0] always succeeds if only low < x. - * - x=0 is also treated as the maximum value, so 0 in (low, high] fails except if high=0. - * - note: 0 in (n, 0] succeeds for any n! + * - high=0 is treated as the maximum value, so `x in (low, 0]` always succeeds if only low < x. + * - x=0 is also treated as the maximum value, so `0 in (low, high]` fails except if high=0. + * - note: `0 in (n, 0]` succeeds for any n! */ -function assertInRange(low: Field, x: Field, high: Field, message?: string) { +function assertBetween(low: Field, x: Field, high: Field, message?: string) { // for low < x, we need to handle the x=0 case separately let xIsZero = x.equals(0n); let lowSafe = Provable.witness(Field, () => (xIsZero.toBoolean() ? 0n : low)); From f41cfa1bd861a6ceddd4113ccacdff960ff17886 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 21:54:28 +0200 Subject: [PATCH 78/93] more docs --- src/lib/provable/merkle-tree-indexed.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index a1ed19103b..1c1999857e 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -49,6 +49,10 @@ export { Leaf }; * } * }) * ``` + * + * Initially, every `IndexedMerkleMap` is populated by a single key-value pair: `(0, 0)`. The value for key `0` can be updated like any other. + * When keys and values are hash outputs, `(0, 0)` can serve as a convenient way to represent a dummy update to the tree, since 0 is not + * effciently computable as a hash image, and this update doesn't affect the Merkle root. */ function IndexedMerkleMap(height: number): typeof IndexedMerkleMapBase { assert(height > 0, 'height must be positive'); From ecb074f075ce553b996bcf22d6032b2681079268 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 3 Jun 2024 21:57:19 +0200 Subject: [PATCH 79/93] another test --- src/lib/provable/test/merkle-tree.unit-test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/provable/test/merkle-tree.unit-test.ts b/src/lib/provable/test/merkle-tree.unit-test.ts index 515a02ad42..b2737dcfe6 100644 --- a/src/lib/provable/test/merkle-tree.unit-test.ts +++ b/src/lib/provable/test/merkle-tree.unit-test.ts @@ -111,6 +111,9 @@ console.log( initialTree.setLeaf(0n, Leaf.hashNode(IndexedMerkleMap(3)._firstLeaf)); expect(map.root).toEqual(initialTree.getRoot()); + // the initial value at key 0 is 0 + expect(map.getOption(0n).assertSome().toBigInt()).toEqual(0n); + map.insert(-1n, 14n); // -1 is the largest possible value map.insert(1n, 13n); From 625619e3bd833796a02acb3421ea5c6ece3b8830 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 09:45:27 +0200 Subject: [PATCH 80/93] methods to mutate imm in place --- src/lib/provable/merkle-tree-indexed.ts | 55 +++++++++++++++++++------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 1c1999857e..e10e89c127 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -102,19 +102,6 @@ class IndexedMerkleMapBase { readonly sortedLeaves: StoredLeaf[]; }>; - clone() { - let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); - cloned.root = this.root; - cloned.length = this.length; - cloned.data.updateAsProver(({ nodes, sortedLeaves }) => { - return { - nodes: nodes.map((row) => [...row]), - sortedLeaves: [...sortedLeaves], - }; - }); - return cloned; - } - // we'd like to do `abstract static provable` here but that's not supported static provable: Provable< IndexedMerkleMapBase, @@ -150,6 +137,48 @@ class IndexedMerkleMapBase { index: 0, }; + /** + * Clone the entire Merkle map. + * + * This method is provable. + */ + clone() { + let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); + cloned.root = this.root; + cloned.length = this.length; + cloned.data.updateAsProver(({ nodes, sortedLeaves }) => { + return { + nodes: nodes.map((row) => [...row]), + sortedLeaves: [...sortedLeaves], + }; + }); + return cloned; + } + + /** + * Overwrite the entire Merkle map with another one. + * + * This method is provable. + */ + overwrite(other: IndexedMerkleMapBase) { + this.overwriteIf(true, other); + } + + /** + * Overwrite the entire Merkle map with another one, if the condition is true. + * + * This method is provable. + */ + overwriteIf(condition: Bool | boolean, other: IndexedMerkleMapBase) { + condition = Bool(condition); + + this.root = Provable.if(condition, other.root, this.root); + this.length = Provable.if(condition, other.length, this.length); + this.data.updateAsProver(() => + Bool(condition).toBoolean() ? other.clone().data.get() : this.data.get() + ); + } + /** * Insert a new leaf `(key, value)`. * From 997dbf4081005b3215a03d4f232890a46631cd98 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 09:47:16 +0200 Subject: [PATCH 81/93] start refactoring offchain state to imm --- src/lib/mina/actions/offchain-state-rollup.ts | 126 +++++++----------- .../actions/offchain-state-serialization.ts | 26 ++-- 2 files changed, 63 insertions(+), 89 deletions(-) diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index 32b91f043d..1f007d04db 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -1,9 +1,8 @@ import { Proof, ZkProgram } from '../../proof-system/zkprogram.js'; import { Bool, Field } from '../../provable/wrapped.js'; -import { Unconstrained } from '../../provable/types/unconstrained.js'; import { MerkleList, MerkleListIterator } from '../../provable/merkle-list.js'; import { Actions } from '../../../bindings/mina-transaction/transaction-leaves.js'; -import { MerkleTree, MerkleWitness } from '../../provable/merkle-tree.js'; +import { IndexedMerkleMap } from '../../provable/merkle-tree-indexed.js'; import { Struct } from '../../provable/types/struct.js'; import { SelfProof } from '../../proof-system/zkprogram.js'; import { Provable } from '../../provable/provable.js'; @@ -27,8 +26,9 @@ class ActionIterator extends MerkleListIterator.create( Actions.emptyActionState() ) {} -const TREE_HEIGHT = 256; -class MerkleMapWitness extends MerkleWitness(TREE_HEIGHT) {} +// TODO: make 31 a parameter +const TREE_HEIGHT = 31; +class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} /** * Commitments that keep track of the current state of an offchain Merkle tree constructed from actions. @@ -46,7 +46,7 @@ class OffchainStateCommitments extends Struct({ actionState: Field, }) { static empty() { - let emptyMerkleRoot = new MerkleTree(TREE_HEIGHT).getRoot(); + let emptyMerkleRoot = new IndexedMerkleMap31().root; return new OffchainStateCommitments({ root: emptyMerkleRoot, actionState: Actions.emptyActionState(), @@ -69,7 +69,7 @@ function merkleUpdateBatch( }, stateA: OffchainStateCommitments, actions: ActionIterator, - tree: Unconstrained + tree: IndexedMerkleMap31 ): OffchainStateCommitments { // this would be unnecessary if the iterator could just be the public input actions.currentHash.assertEquals(stateA.actionState); @@ -98,82 +98,55 @@ function merkleUpdateBatch( } actions.assertAtEnd(); - // update merkle root at once for the actions of each account update - let root = stateA.root; - let intermediateRoot = root; - - let intermediateUpdates: { key: Field; value: Field }[] = []; - let intermediateTree = Unconstrained.witness(() => tree.get().clone()); + // tree must match the public Merkle root; the method operates on the tree internally + // TODO: this would be simpler if the tree was the public input direcetly + stateA.root.assertEquals(tree.root); + let initialTree = tree; + let intermediateTree = tree.clone(); let isValidUpdate = Bool(true); linearActions.forEach(maxActionsPerBatch, (element, isDummy) => { let { action, isCheckPoint } = element; let { key, value, usesPreviousValue, previousValue } = action; - // merkle witness - let witness = Provable.witness( - MerkleMapWitness, - () => - new MerkleMapWitness(intermediateTree.get().getWitness(key.toBigInt())) - ); - - // previous value at the key - let actualPreviousValue = Provable.witness(Field, () => - intermediateTree.get().getLeaf(key.toBigInt()) - ); + // make sure that if this is a dummy action, we use the canonical dummy (key, value) pair + key = Provable.if(isDummy, Field(0n), key); + value = Provable.if(isDummy, Field(0n), value); - // prove that the witness and `actualPreviousValue` is correct, by comparing the implied root and key - // note: this just works if the (key, value) is a (0,0) dummy, because the value at the 0 key will always be 0 - witness.calculateIndex().assertEquals(key, 'key mismatch'); - witness - .calculateRoot(actualPreviousValue) - .assertEquals(intermediateRoot, 'root mismatch'); + // set (key, value) in the intermediate tree + // note: this just works if (key, value) is a (0,0) dummy, because the value at the 0 key will always be 0 + let actualPreviousValue = intermediateTree.set(key, value); // if an expected previous value was provided, check whether it matches the actual previous value // otherwise, the entire update in invalidated - let matchesPreviousValue = actualPreviousValue.equals(previousValue); + let matchesPreviousValue = actualPreviousValue + .orElse(0n) + .equals(previousValue); let isValidAction = usesPreviousValue.implies(matchesPreviousValue); isValidUpdate = isValidUpdate.and(isValidAction); - // store new value in at the key - let newRoot = witness.calculateRoot(value); - - // update intermediate root if this wasn't a dummy action - intermediateRoot = Provable.if(isDummy, intermediateRoot, newRoot); - - // at checkpoints, update the root, if the entire update was valid - root = Provable.if(isCheckPoint.and(isValidUpdate), intermediateRoot, root); + // at checkpoints, update the tree, if the entire update was valid + tree = Provable.if( + isCheckPoint.and(isValidUpdate), + IndexedMerkleMap31.provable, + intermediateTree, + tree + ); // at checkpoints, reset intermediate values - let wasValidUpdate = isValidUpdate; isValidUpdate = Provable.if(isCheckPoint, Bool(true), isValidUpdate); - intermediateRoot = Provable.if(isCheckPoint, root, intermediateRoot); - - // update the tree, outside the circuit (this should all be part of a better merkle tree API) - Provable.asProver(() => { - // ignore dummy value - if (isDummy.toBoolean()) return; - - intermediateTree.get().setLeaf(key.toBigInt(), value.toConstant()); - intermediateUpdates.push({ key, value }); - - if (isCheckPoint.toBoolean()) { - // if the update was valid, apply the intermediate updates to the actual tree - if (wasValidUpdate.toBoolean()) { - intermediateUpdates.forEach(({ key, value }) => { - tree.get().setLeaf(key.toBigInt(), value.toConstant()); - }); - } - // otherwise, we have to roll back the intermediate tree (TODO: inefficient) - else { - intermediateTree.set(tree.get().clone()); - } - intermediateUpdates = []; - } - }); + intermediateTree = Provable.if( + isCheckPoint, + IndexedMerkleMap31.provable, + tree, + intermediateTree + ); }); - return { root, actionState: actions.currentHash }; + // mutate the input tree with the final state + initialTree.overwrite(tree); + + return { root: tree.root, actionState: actions.currentHash }; } /** @@ -198,12 +171,12 @@ function OffchainStateRollup({ */ firstBatch: { // [actions, tree] - privateInputs: [ActionIterator.provable, Unconstrained.provable], + privateInputs: [ActionIterator.provable, IndexedMerkleMap31.provable], async method( stateA: OffchainStateCommitments, actions: ActionIterator, - tree: Unconstrained + tree: IndexedMerkleMap31 ): Promise { return merkleUpdateBatch( { maxActionsPerBatch, maxActionsPerUpdate }, @@ -220,14 +193,14 @@ function OffchainStateRollup({ // [actions, tree, proof] privateInputs: [ ActionIterator.provable, - Unconstrained.provable, + IndexedMerkleMap31.provable, SelfProof, ], async method( stateA: OffchainStateCommitments, actions: ActionIterator, - tree: Unconstrained, + tree: IndexedMerkleMap31, recursiveProof: Proof< OffchainStateCommitments, OffchainStateCommitments @@ -271,7 +244,10 @@ function OffchainStateRollup({ return result; }, - async prove(tree: MerkleTree, actions: MerkleList>) { + async prove( + tree: IndexedMerkleMap31, + actions: MerkleList> + ) { assert(tree.height === TREE_HEIGHT, 'Tree height must match'); if (getProofsEnabled()) await this.compile(); // clone the tree so we don't modify the input @@ -280,7 +256,7 @@ function OffchainStateRollup({ // input state let iterator = actions.startIterating(); let inputState = new OffchainStateCommitments({ - root: tree.getRoot(), + root: tree.root, actionState: iterator.currentHash, }); @@ -303,7 +279,7 @@ function OffchainStateRollup({ updateMerkleMap(actionsList, tree); let finalState = new OffchainStateCommitments({ - root: tree.getRoot(), + root: tree.root, actionState: iterator.hash, }); let proof = await RollupProof.dummy(inputState, finalState, 2, 15); @@ -312,11 +288,7 @@ function OffchainStateRollup({ // base proof let slice = sliceActions(iterator, maxActionsPerBatch); - let proof = await offchainStateRollup.firstBatch( - inputState, - slice, - Unconstrained.from(tree) - ); + let proof = await offchainStateRollup.firstBatch(inputState, slice, tree); // recursive proofs let nProofs = 1; @@ -328,7 +300,7 @@ function OffchainStateRollup({ proof = await offchainStateRollup.nextBatch( inputState, slice, - Unconstrained.from(tree), + tree, proof ); } diff --git a/src/lib/mina/actions/offchain-state-serialization.ts b/src/lib/mina/actions/offchain-state-serialization.ts index 61d7d3d032..c336ab4e7e 100644 --- a/src/lib/mina/actions/offchain-state-serialization.ts +++ b/src/lib/mina/actions/offchain-state-serialization.ts @@ -24,8 +24,8 @@ import * as Mina from '../mina.js'; import { PublicKey } from '../../provable/crypto/signature.js'; import { Provable } from '../../provable/provable.js'; import { Actions } from '../account-update.js'; -import { MerkleTree } from '../../provable/merkle-tree.js'; import { Option } from '../../provable/option.js'; +import { IndexedMerkleMap } from '../../provable/merkle-tree-indexed.js'; export { toKeyHash, @@ -44,6 +44,9 @@ export { type Action = [...Field[], Field, Field]; type Actionable = ProvableHashable & ProvablePure; +// TODO: make 31 a parameter +class IndexedMerkleMap31 extends IndexedMerkleMap(31) {} + function toKeyHash | undefined>( prefix: Field, keyType: KeyType, @@ -258,7 +261,7 @@ async function fetchMerkleLeaves( async function fetchMerkleMap( contract: { address: PublicKey; tokenId: Field }, endActionState?: Field -): Promise<{ merkleMap: MerkleTree; valueMap: Map }> { +): Promise<{ merkleMap: IndexedMerkleMap31; valueMap: Map }> { let result = await Mina.fetchActions( contract.address, { endActionState }, @@ -272,7 +275,7 @@ async function fetchMerkleMap( .reverse() ); - let merkleMap = new MerkleTree(256); + let merkleMap = new IndexedMerkleMap31(); let valueMap = new Map(); updateMerkleMap(leaves, merkleMap, valueMap); @@ -282,14 +285,14 @@ async function fetchMerkleMap( function updateMerkleMap( updates: MerkleLeaf[][], - tree: MerkleTree, + tree: IndexedMerkleMap31, valueMap?: Map ) { let intermediateTree = tree.clone(); for (let leaves of updates) { let isValidUpdate = true; - let updates: { key: bigint; value: bigint; fullValue: Field[] }[] = []; + let updates: { key: bigint; fullValue: Field[] }[] = []; for (let leaf of leaves) { let { key, value, usesPreviousValue, previousValue, prefix } = @@ -298,28 +301,27 @@ function updateMerkleMap( // the update is invalid if there is an unsatisfied precondition let isValidAction = !usesPreviousValue || - intermediateTree.getLeaf(key).toBigInt() === previousValue; + intermediateTree.get(key).toBigInt() === previousValue; if (!isValidAction) { isValidUpdate = false; - break; } // update the intermediate tree, save updates for final tree - intermediateTree.setLeaf(key, Field(value)); - updates.push({ key, value, fullValue: prefix.get() }); + intermediateTree.set(key, value); + updates.push({ key, fullValue: prefix.get() }); } if (isValidUpdate) { // if the update was valid, we can commit the updates - for (let { key, value, fullValue } of updates) { - tree.setLeaf(key, Field(value)); + tree.overwrite(intermediateTree); + for (let { key, fullValue } of updates) { if (valueMap) valueMap.set(key, fullValue); } } else { // if the update was invalid, we have to roll back the intermediate tree - intermediateTree = tree.clone(); + intermediateTree.overwrite(tree); } } } From cd2e03de8b47654c74ac3b42aea8949375e8a0f6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 10:11:48 +0200 Subject: [PATCH 82/93] finish refactoring offchain state --- src/lib/mina/actions/offchain-state.ts | 41 +++++++++++++------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/lib/mina/actions/offchain-state.ts b/src/lib/mina/actions/offchain-state.ts index c0582b4e35..e40866f87f 100644 --- a/src/lib/mina/actions/offchain-state.ts +++ b/src/lib/mina/actions/offchain-state.ts @@ -22,7 +22,7 @@ import { Actions } from '../account-update.js'; import { Provable } from '../../provable/provable.js'; import { Poseidon } from '../../provable/crypto/poseidon.js'; import { smartContractContext } from '../smart-contract-context.js'; -import { MerkleTree, MerkleWitness } from '../../provable/merkle-tree.js'; +import { IndexedMerkleMap } from '../../provable/merkle-tree-indexed.js'; export { OffchainState, OffchainStateCommitments }; @@ -98,7 +98,7 @@ type OffchainStateContract = SmartContract & { offchainState: State; }; -const MerkleWitness256 = MerkleWitness(256); +class IndexedMerkleMap31 extends IndexedMerkleMap(31) {} /** * Offchain state for a `SmartContract`. @@ -135,13 +135,13 @@ function OffchainState< // setup internal state of this "class" let internal = { _contract: undefined as OffchainStateContract | undefined, - _merkleMap: undefined as MerkleTree | undefined, + _merkleMap: undefined as IndexedMerkleMap31 | undefined, _valueMap: undefined as Map | undefined, get contract() { assert( internal._contract !== undefined, - 'Must call `setContractAccount()` first' + 'Must call `setContractInstance()` first' ); return internal._contract; }, @@ -189,7 +189,17 @@ function OffchainState< // get onchain merkle root let stateRoot = contract().offchainState.getAndRequireEquals().root; - // witness the actual value + // witness the merkle map & anchor against the onchain root + let map = await Provable.witnessAsync( + IndexedMerkleMap31.provable, + async () => (await merkleMaps()).merkleMap + ); + map.root.assertEquals(stateRoot, 'root mismatch'); + + // get the value hash + let valueHash = map.getOption(key); + + // witness the full value const optionType = Option(valueType); let value = await Provable.witnessAsync(optionType, async () => { let { valueMap } = await merkleMaps(); @@ -201,23 +211,12 @@ function OffchainState< return optionType.from(value); }); - // witness a merkle witness - let witness = await Provable.witnessAsync(MerkleWitness256, async () => { - let { merkleMap } = await merkleMaps(); - return new MerkleWitness256(merkleMap.getWitness(key.toBigInt())); - }); - - // anchor the value against the onchain root and passed in key - // we also allow the value to be missing, in which case the map must contain the 0 element - let valueHash = Provable.if( - value.isSome, - Poseidon.hashPacked(valueType, value.value), - Field(0) + // assert that the value hash matches the value, or both are none + let hashMatches = Poseidon.hashPacked(valueType, value.value).equals( + valueHash.value ); - let actualKey = witness.calculateIndex(); - let actualRoot = witness.calculateRoot(valueHash); - key.assertEquals(actualKey, 'key mismatch'); - stateRoot.assertEquals(actualRoot, 'root mismatch'); + let bothNone = value.isSome.or(valueHash.isSome).not(); + assert(hashMatches.or(bothNone), 'value hash mismatch'); return value; } From 1fdeb520f3fdd309e92b5100c9828db6d0d30427 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 12:49:07 +0200 Subject: [PATCH 83/93] recompute good defaults --- src/lib/mina/actions/offchain-state-rollup.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index 1f007d04db..051d2a401d 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -153,13 +153,20 @@ function merkleUpdateBatch( * This program represents a proof that we can go from OffchainStateCommitments A -> B */ function OffchainStateRollup({ - // 1 action uses about 7.5k constraints - // we can fit at most 7 * 7.5k = 52.5k constraints in one method next to proof verification - // => we use `maxActionsPerBatch = 6` to safely stay below the constraint limit - // the second parameter `maxActionsPerUpdate` only weakly affects # constraints, but has to be <= `maxActionsPerBatch` - // => so we set it to the same value - maxActionsPerBatch = 6, - maxActionsPerUpdate = 6, + /** + * the constraints used in one batch proof with a height-31 tree are: + * + * 1967*A + 87*A*U + 2 + * + * where A = maxActionsPerBatch and U = maxActionsPerUpdate. + * + * To determine defaults, we set U=4 which should cover most use cases while ensuring + * that the main loop which is independent of U dominates. + * + * Targeting ~50k constraints, to leave room for recursive verification, yields A=22. + */ + maxActionsPerBatch = 22, + maxActionsPerUpdate = 4, } = {}) { let offchainStateRollup = ZkProgram({ name: 'merkle-map-rollup', From 546cd2343917c408e10b8dcfff38f62e328e3e37 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 13:59:04 +0200 Subject: [PATCH 84/93] fix clone() --- src/lib/provable/merkle-tree-indexed.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index e10e89c127..f65f30ed71 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -146,7 +146,8 @@ class IndexedMerkleMapBase { let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); cloned.root = this.root; cloned.length = this.length; - cloned.data.updateAsProver(({ nodes, sortedLeaves }) => { + cloned.data.updateAsProver(() => { + let { nodes, sortedLeaves } = this.data.get(); return { nodes: nodes.map((row) => [...row]), sortedLeaves: [...sortedLeaves], From f5268fd3ed40228cc2fe4cb38aa6e23c12144fcd Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 14:01:01 +0200 Subject: [PATCH 85/93] make tree height a constant, small fix merkle update logic, small refactor --- src/lib/mina/actions/offchain-state-rollup.ts | 21 ++++--------------- .../actions/offchain-state-serialization.ts | 8 ++++--- src/lib/mina/actions/offchain-state.ts | 3 ++- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index 051d2a401d..e673de03eb 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -12,6 +12,7 @@ import { LinearizedAction, LinearizedActionList, MerkleLeaf, + TREE_HEIGHT, updateMerkleMap, } from './offchain-state-serialization.js'; import { getProofsEnabled } from '../mina.js'; @@ -27,7 +28,6 @@ class ActionIterator extends MerkleListIterator.create( ) {} // TODO: make 31 a parameter -const TREE_HEIGHT = 31; class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} /** @@ -102,7 +102,6 @@ function merkleUpdateBatch( // TODO: this would be simpler if the tree was the public input direcetly stateA.root.assertEquals(tree.root); - let initialTree = tree; let intermediateTree = tree.clone(); let isValidUpdate = Bool(true); @@ -127,25 +126,13 @@ function merkleUpdateBatch( isValidUpdate = isValidUpdate.and(isValidAction); // at checkpoints, update the tree, if the entire update was valid - tree = Provable.if( - isCheckPoint.and(isValidUpdate), - IndexedMerkleMap31.provable, - intermediateTree, - tree - ); + tree.overwriteIf(isCheckPoint.and(isValidUpdate), intermediateTree); + // at checkpoints, reset intermediate values isValidUpdate = Provable.if(isCheckPoint, Bool(true), isValidUpdate); - intermediateTree = Provable.if( - isCheckPoint, - IndexedMerkleMap31.provable, - tree, - intermediateTree - ); + intermediateTree.overwriteIf(isCheckPoint, tree); }); - // mutate the input tree with the final state - initialTree.overwrite(tree); - return { root: tree.root, actionState: actions.currentHash }; } diff --git a/src/lib/mina/actions/offchain-state-serialization.ts b/src/lib/mina/actions/offchain-state-serialization.ts index c336ab4e7e..38f435010d 100644 --- a/src/lib/mina/actions/offchain-state-serialization.ts +++ b/src/lib/mina/actions/offchain-state-serialization.ts @@ -39,13 +39,15 @@ export { fetchMerkleMap, updateMerkleMap, Actionable, + TREE_HEIGHT, }; type Action = [...Field[], Field, Field]; type Actionable = ProvableHashable & ProvablePure; // TODO: make 31 a parameter -class IndexedMerkleMap31 extends IndexedMerkleMap(31) {} +const TREE_HEIGHT = 31; +class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} function toKeyHash | undefined>( prefix: Field, @@ -299,9 +301,9 @@ function updateMerkleMap( MerkleLeaf.toValue(leaf); // the update is invalid if there is an unsatisfied precondition + let previous = intermediateTree.getOption(key).orElse(0n); let isValidAction = - !usesPreviousValue || - intermediateTree.get(key).toBigInt() === previousValue; + !usesPreviousValue || previous.toBigInt() === previousValue; if (!isValidAction) { isValidUpdate = false; diff --git a/src/lib/mina/actions/offchain-state.ts b/src/lib/mina/actions/offchain-state.ts index e40866f87f..dca6093ade 100644 --- a/src/lib/mina/actions/offchain-state.ts +++ b/src/lib/mina/actions/offchain-state.ts @@ -1,6 +1,7 @@ import { InferProvable } from '../../provable/types/struct.js'; import { Actionable, + TREE_HEIGHT, fetchMerkleLeaves, fetchMerkleMap, fromActionWithoutHashes, @@ -98,7 +99,7 @@ type OffchainStateContract = SmartContract & { offchainState: State; }; -class IndexedMerkleMap31 extends IndexedMerkleMap(31) {} +class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} /** * Offchain state for a `SmartContract`. From 4a659c807188d05d378d2b28d200ae5a03d6900e Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 4 Jun 2024 08:55:21 -0700 Subject: [PATCH 86/93] refactor(foreign-ecdsa.ts, elliptic-curve.ts): remove deprecated methods and their documentation --- src/lib/provable/crypto/foreign-ecdsa.ts | 37 ---------------------- src/lib/provable/gadgets/elliptic-curve.ts | 14 -------- 2 files changed, 51 deletions(-) diff --git a/src/lib/provable/crypto/foreign-ecdsa.ts b/src/lib/provable/crypto/foreign-ecdsa.ts index fdef048a0f..0b74d0291f 100644 --- a/src/lib/provable/crypto/foreign-ecdsa.ts +++ b/src/lib/provable/crypto/foreign-ecdsa.ts @@ -67,38 +67,6 @@ class EcdsaSignature { } /** - * Verify the ECDSA signature given the message (an array of bytes) and public key (a {@link Curve} point). - * - * **Important:** This method returns a {@link Bool} which indicates whether the signature is valid. - * So, to actually prove validity of a signature, you need to assert that the result is true. - * - * @throws if one of the signature scalars is zero or if the public key is not on the curve. - * - * @example - * ```ts - * // create classes for your curve - * class Secp256k1 extends createForeignCurve(Crypto.CurveParams.Secp256k1) {} - * class Scalar extends Secp256k1.Scalar {} - * class Ecdsa extends createEcdsa(Secp256k1) {} - * - * let message = 'my message'; - * let messageBytes = new TextEncoder().encode(message); - * - * // outside provable code: create inputs - * let privateKey = Scalar.random(); - * let publicKey = Secp256k1.generator.scale(privateKey); - * let signature = Ecdsa.sign(messageBytes, privateKey.toBigInt()); - * - * // ... - * // in provable code: create input witnesses (or use method inputs, or constants) - * let pk = Provable.witness(Secp256k1.provable, () => publicKey); - * let msg = Provable.witness(Provable.Array(Field, 9), () => messageBytes.map(Field)); - * let sig = Provable.witness(Ecdsa.provable, () => signature); - * - * // verify signature - * let isValid = sig.verify(msg, pk); - * isValid.assertTrue('signature verifies'); - * ``` * @deprecated There is a security vulnerability in this method. Use {@link verifyV2} instead. */ verify(message: Bytes, publicKey: FlexiblePoint) { @@ -148,11 +116,6 @@ class EcdsaSignature { } /** - * Verify the ECDSA signature given the message hash (a {@link Scalar}) and public key (a {@link Curve} point). - * - * This is a building block of {@link EcdsaSignature.verify}, where the input message is also hashed. - * In contrast, this method just takes the message hash (a curve scalar) as input, giving you flexibility in - * choosing the hashing algorithm. * @deprecated There is a security vulnerability in this method. Use {@link verifySignedHashV2} instead. */ verifySignedHash( diff --git a/src/lib/provable/gadgets/elliptic-curve.ts b/src/lib/provable/gadgets/elliptic-curve.ts index aec08d80bd..7e4ca45492 100644 --- a/src/lib/provable/gadgets/elliptic-curve.ts +++ b/src/lib/provable/gadgets/elliptic-curve.ts @@ -285,20 +285,6 @@ function verifyEcdsaGeneric( } /** - * Verify an ECDSA signature. - * - * Details about the `config` parameter: - * - For both the generator point `G` and public key `P`, `config` allows you to specify: - * - the `windowSize` which is used in scalar multiplication for this point. - * this flexibility is good because the optimal window size is different for constant and non-constant points. - * empirically, `windowSize=4` for constants and 3 for variables leads to the fewest constraints. - * our defaults reflect that the generator is always constant and the public key is variable in typical applications. - * - a table of multiples of those points, of length `2^windowSize`, which is used in the scalar multiplication gadget to speed up the computation. - * if these are not provided, they are computed on the fly. - * for the constant G, computing multiples costs no constraints, so passing them in makes no real difference. - * for variable public key, there is a possible use case: if the public key is a public input, then its multiples could also be. - * in that case, passing them in would avoid computing them in-circuit and save a few constraints. - * - The initial aggregator `ia`, see {@link initialAggregator}. By default, `ia` is computed deterministically on the fly. * @deprecated There is a security vulnerability with this function, allowing a prover to modify witness values than what is expected. Use {@link verifyEcdsaV2} instead. */ function verifyEcdsa( From cdea8bcda741f4bb71b14b4dfdd394d277d0ea66 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 4 Jun 2024 08:55:33 -0700 Subject: [PATCH 87/93] fix(elliptic-curve.ts): adjust windowSize parameter from 4 to 3 in P object to optimize elliptic curve calculations --- src/lib/provable/gadgets/elliptic-curve.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/provable/gadgets/elliptic-curve.ts b/src/lib/provable/gadgets/elliptic-curve.ts index 7e4ca45492..58472e8c00 100644 --- a/src/lib/provable/gadgets/elliptic-curve.ts +++ b/src/lib/provable/gadgets/elliptic-curve.ts @@ -334,7 +334,7 @@ function verifyEcdsaV2( G?: { windowSize: number; multiples?: Point[] }; P?: { windowSize: number; multiples?: Point[] }; ia?: point; - } = { G: { windowSize: 4 }, P: { windowSize: 4 } } + } = { G: { windowSize: 4 }, P: { windowSize: 3 } } ) { return verifyEcdsaGeneric( Curve, From 14c54ef5b807a8a18eff234d4a09028f23316588 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 4 Jun 2024 08:55:43 -0700 Subject: [PATCH 88/93] refactor(elliptic-curve.ts): move hashedTables initialization inside hashed condition to optimize memory usage and improve performance --- src/lib/provable/gadgets/elliptic-curve.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/lib/provable/gadgets/elliptic-curve.ts b/src/lib/provable/gadgets/elliptic-curve.ts index 58472e8c00..afab631f6e 100644 --- a/src/lib/provable/gadgets/elliptic-curve.ts +++ b/src/lib/provable/gadgets/elliptic-curve.ts @@ -504,16 +504,19 @@ function multiScalarMul( // a Point is 6 field elements, the hash is just 1 field element const HashedPoint = Hashed.create(Point.provable); - let hashedTables = tables.map((table) => - table.map((point) => HashedPoint.hash(point)) - ); - // initialize sum to the initial aggregator, which is expected to be unrelated to any point that this gadget is used with // note: this is a trick to ensure _completeness_ of the gadget // soundness follows because add() and double() are sound, on all inputs that are valid non-zero curve points ia ??= initialAggregator(Curve); let sum = Point.from(ia); + let hashedTables: Hashed[][] = []; + if (hashed) { + hashedTables = tables.map((table) => + table.map((point) => HashedPoint.hash(point)) + ); + } + for (let i = maxBits - 1; i >= 0; i--) { // add in multiple of each point for (let j = 0; j < n; j++) { From 39df06e5adf720bfa856ced0b38477853d8fddf4 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 4 Jun 2024 09:00:41 -0700 Subject: [PATCH 89/93] chore(vk-regression): update vk-regression artifacts --- tests/vk-regression/vk-regression.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/vk-regression/vk-regression.json b/tests/vk-regression/vk-regression.json index d80e7a6fc8..db07d5bd82 100644 --- a/tests/vk-regression/vk-regression.json +++ b/tests/vk-regression/vk-regression.json @@ -262,29 +262,29 @@ } }, "ecdsa-only": { - "digest": "3e306cbba2e987fe302992a81aba9551d47df260f7a3b8b8abeee6bc938d7", + "digest": "3981c6b30a5d6eccbdd26351e5e059cbbee162f43a4cd6280b2335e66c8ae9ba", "methods": { "verifySignedHash": { - "rows": 34257, - "digest": "0d84b4072ac7aeb3f387d1b03b31809f" + "rows": 31689, + "digest": "87cdb8324830be5385ab306e90c4bad2" } }, "verificationKey": { - "data": "AAD0TiJIvE46IEuFjZed3VZt7S8Wp0kRCJXb3a2Ju7WUBunBgHb4SXvz1c3QjP2nd1qSYoUr66taz9IKVgu+5so8TiBnYTRcd7RsaAjbdbIbQJ9EuopFRFewZRx9qeQeEibNeMRcRMP4LdfS3AQRxhFZzN4HFa4MbtGs+Aja820cIw5azWYCxKf0pWJRDHbLUtJrjnQT0ATHD77rtbgLedcfFKfCQS5oAbF7hIhCbAsm7wMT+9+ZX8M5354UKZ03NiLUh5XNLAlGgVi7FfWR6p9P72AAymyD3lUdecJyZmCREiVgPrTdFppkp45TefJWNTySkV9c5YzpNxQoXedZDvYP/5s4KBkfIeK+zB2yJC9eZ1ZDYfM88shGDYxmBtur9AkQ49QGquR+kYUI0lpXtuNMG+ZRy0FRJ8ci/TE+PIPIFnSiGcSOA3YM2G171LYf89abU2QUoQRHSP3PmmAOy/8CoRLVro7Nl6z/Ou0oZzX7RjOEo//LBqcSWa2S9X8TQz0R3uivbovTdq0rrba56SbEnK6LWItmBc6CubYWL7UzDbD3RZM6iRz1hqTHDzDz7UIWOzHgLqW9rjnZllQCyfsSAAs+mZNp6eMcA9nsWiKZeNIjKS/efiDZVKA0XR4LNUoAy2KsbYbk7XBknDhh0/POjv9ThJ81GTAksHfHEjyjPAcgKzcNZhhPW5VfbcSYDpx5nVaU5pTEFl+2+RlcuhBpG1ksAWbD64AUKDjdyTWIC5Wn68AagPtG65V13eFS5LgkSfVtNXxGodg7SdP4AJmXpBgZfzMg4RW6Qje5ZFfrwRzoHPo0y7nO1hkaNLGV3Wvd3/pYiebXvyo+DdTZmaMbJpJaGSCysnovOrVUIpcn4h1hvA12jztQFQcbNHoVeZgslPxA54y9ynjhN7VZfT8lNXXIrRCpmPaxZW6Bw6Op/g6P1Y8pKZHixBy1UrxqWGI+49oRtRFGw9CWS21EekuBFeu9RKI6yZLDiyRC2b3koFG+Kp6oq5Ej6Q8uargE09Ago9Z2Ak+8OPG2V17zCdK2baF3CgW/HuWQT3dmWCesPz2nk99eNiSJzx2lLOFaWlp8VCG02On55KRZInJzhIEMP146ddCU9VJ1JmYsYa+MYEgKjZCvABbX9AEY2ggMr1cHaA49GrGul+Sj6pAvz4oyzaR8m7WAPMDuBtVwdbDtfju3CVbhX15uBoKhuYWQgLr2rnVJ5SOZoDvlwJtcK2izLMYVAasejw4fvsehYGb88wvDbFxS6sM9gDSgTlavZRs95Qf+c1KpYf/jb8BxYNrwrqy8F++c1APDzfzQ/IbVLiaL28wkEy412qmXSjM+9hErKXFy8JIT/WBOIWMMg/7mMu1Al7Tt/kOZrDznlS/szLlpAp2jISa8VWCmlAEPrustdNqQvptSsF6hikzXZVXg5f8pU4Gpa0TP0TRFvIYfmTyl8HpdFOG7vpQavC600YgzS2YGtY7K2WQ5GtN5ZTZBHPsUSir2yKSo9Le9CWXbDtn3SBDepWypwDa3YWKtNog+y10VmpL1N+RG3u1DXSuY7y9WZgkQ7tdvyx/Gjr91kjF0s3bt7vHIAZCtzNlRlWDBz3og0cSnEucCEuKR6dL2Mz+RuF1GmLoXZXapUjVG/82BjdAMAOxPlE67lEs+JWgnrVrA5NLJoL4DZ6+fhQKpNfk0uOrEfZIWR9Sau0IBwBxu6IYVm5/XAB19dt8MAuVcRdN/JGGzo0Hr3WVJuKzbAhuFwJZzcd1J1n4xO09ECT5NQdFSFXGsy8kIFjRNEOkLl+bAExePtGCt0w6cYqB0uCeX3lTI7ugIEgdStMtHFiWngJ218l8CuVrkwTJ7ZqHLtuJDiNqlLptkHWChDfw+IgDwz85dZrfBBzQrMRWranxQmisM+wx3vC+pLURRQHZJEasGCAElj0lTColrqQ/cXS7cBaqs1tBsQDGzKYMCMwsqL53fyxGCljVvljBa99+FpYfoUK+Fi0z6uEbem+luXRScr2yPB5I08lnBY23RmBb/pfSyBfbcmnmF5BkRlJTJKY7fQL/t9bFfywoquQe9e7OQvIjppA/FO7HmZS6hoOU+eS8+W94fEF2gvrowpTeqQHM6hLN9Qzl8niwZWUIyRCfyuzQnuSz/VP1K2sMFBKnZZNDcuBh1/xSFymOH6LfNKostvc6qHTIxrTjlH6952bo1bQl+mVvBUaJuRkYh12QbcyIyzcBFUYwaFazzkHXMof0O30oL3Q6wegTvJxTSZD5VCr5D26Myzoa0JBpqL0st9/MNGZe5a/+HW1qan/VtGA5nYkJcUzwKVqqlmZeuOZekFLGxlfp0lv9IQUQWtiU5uvd5HVoolEc/teUnx/IxYe01IDxX9cbmPMJnLYXJGSY=", - "hash": "21569291188270991858171229626958471686538549641430686923686948193105860016913" + "data": "AAD0TiJIvE46IEuFjZed3VZt7S8Wp0kRCJXb3a2Ju7WUBunBgHb4SXvz1c3QjP2nd1qSYoUr66taz9IKVgu+5so8TiBnYTRcd7RsaAjbdbIbQJ9EuopFRFewZRx9qeQeEibNeMRcRMP4LdfS3AQRxhFZzN4HFa4MbtGs+Aja820cIw5azWYCxKf0pWJRDHbLUtJrjnQT0ATHD77rtbgLedcfFKfCQS5oAbF7hIhCbAsm7wMT+9+ZX8M5354UKZ03NiLUh5XNLAlGgVi7FfWR6p9P72AAymyD3lUdecJyZmCREiVgPrTdFppkp45TefJWNTySkV9c5YzpNxQoXedZDvYP/5s4KBkfIeK+zB2yJC9eZ1ZDYfM88shGDYxmBtur9AkQ49QGquR+kYUI0lpXtuNMG+ZRy0FRJ8ci/TE+PIPIFnSiGcSOA3YM2G171LYf89abU2QUoQRHSP3PmmAOy/8CoRLVro7Nl6z/Ou0oZzX7RjOEo//LBqcSWa2S9X8TQz0R3uivbovTdq0rrba56SbEnK6LWItmBc6CubYWL7UzDbD3RZM6iRz1hqTHDzDz7UIWOzHgLqW9rjnZllQCyfsSAHzKVEEoVCOPXn5qzQsEEa7rDqWgLi4KVq5R4NLLt5w/titLcOybRhYCMk4nB3OVJQuC3rQKXPhIyyao8UXiBjwgKzcNZhhPW5VfbcSYDpx5nVaU5pTEFl+2+RlcuhBpG1ksAWbD64AUKDjdyTWIC5Wn68AagPtG65V13eFS5LgkSfVtNXxGodg7SdP4AJmXpBgZfzMg4RW6Qje5ZFfrwRzoHPo0y7nO1hkaNLGV3Wvd3/pYiebXvyo+DdTZmaMbJpJaGSCysnovOrVUIpcn4h1hvA12jztQFQcbNHoVeZgslPxA54y9ynjhN7VZfT8lNXXIrRCpmPaxZW6Bw6Op/g6FGIs8TlruzZJRhz1lOLvl2FPvrUsFtz4yWTPjbT+VGsKuJFPvMuYybxq8pGyWVQN023uObDel47krlcQoH4MAm9ipKXzMO25PK8L3KtA4PPDP+En4qeIrshifHItQjCZDzgtE5oz0hXkEbNiWNzV7alOrR39PD1yuaLyXlx9DEvDR5DLlenSa0wQ3PXdv/C9LpDvkzJOLZs+/ZePd4YMI0+WuP2+6Xas4aNM+4JkNuHF5uMDcxgWID4TUy7Vdlzm3CVbhX15uBoKhuYWQgLr2rnVJ5SOZoDvlwJtcK2izLMYVAasejw4fvsehYGb88wvDbFxS6sM9gDSgTlavZRs95Qf+c1KpYf/jb8BxYNrwrqy8F++c1APDzfzQ/IbVLiaL28wkEy412qmXSjM+9hErKXFy8JIT/WBOIWMMg/7mMjvxngHnci+aYJZ6J+Lszh5zgo708vzO7fwaxC0wgd8anH3gFrbFnOg1hkmmoUEIgIwXh+ynuoZPOaoKNXNm1jOl8HpdFOG7vpQavC600YgzS2YGtY7K2WQ5GtN5ZTZBHPsUSir2yKSo9Le9CWXbDtn3SBDepWypwDa3YWKtNog+y10VmpL1N+RG3u1DXSuY7y9WZgkQ7tdvyx/Gjr91kjF0s3bt7vHIAZCtzNlRlWDBz3og0cSnEucCEuKR6dL2Mz+RuF1GmLoXZXapUjVG/82BjdAMAOxPlE67lEs+JWgnrVrA5NLJoL4DZ6+fhQKpNfk0uOrEfZIWR9Sau0IBwBxu6IYVm5/XAB19dt8MAuVcRdN/JGGzo0Hr3WVJuKzbAhuFwJZzcd1J1n4xO09ECT5NQdFSFXGsy8kIFjRNEOkLl+bAExePtGCt0w6cYqB0uCeX3lTI7ugIEgdStMtHFiWngJ218l8CuVrkwTJ7ZqHLtuJDiNqlLptkHWChDfw+IgDwz85dZrfBBzQrMRWranxQmisM+wx3vC+pLURRQHZJEasGCAElj0lTColrqQ/cXS7cBaqs1tBsQDGzKYMCMwsqL53fyxGCljVvljBa99+FpYfoUK+Fi0z6uEbem+luXRScr2yPB5I08lnBY23RmBb/pfSyBfbcmnmF5BkRlJTJKY7fQL/t9bFfywoquQe9e7OQvIjppA/FO7HmZS6hoOU+eS8+W94fEF2gvrowpTeqQHM6hLN9Qzl8niwZWUIyRCfyuzQnuSz/VP1K2sMFBKnZZNDcuBh1/xSFymOH6LfNKostvc6qHTIxrTjlH6952bo1bQl+mVvBUaJuRkYh12QbcyIyzcBFUYwaFazzkHXMof0O30oL3Q6wegTvJxTSZD5VCr5D26Myzoa0JBpqL0st9/MNGZe5a/+HW1qan/VtGA5nYkJcUzwKVqqlmZeuOZekFLGxlfp0lv9IQUQWtiU5uvd5HVoolEc/teUnx/IxYe01IDxX9cbmPMJnLYXJGSY=", + "hash": "5709505770243633710702038873416963172579507364104803648209582439539220801405" } }, "ecdsa": { - "digest": "d3869b477c270fdef32c71260bfca827f8840666b0d178a5c70606013038411", + "digest": "1324f161aac5b24f90009d845c4f1273a52062fd54b96f69bfaf625ff06b5c8b", "methods": { "verifyEcdsa": { - "rows": 48755, - "digest": "4c8bf67689c0247a32385f3d971ff6ce" + "rows": 46187, + "digest": "f173d91e37a7d9041cef31c3f239a194" } }, "verificationKey": { - "data": "AAClzwJllhsXW4UHH6dHRfysOr2Bv96SCGeRT4Wa3IseIaKiigrgpOaDXTEEjlaJI+fiakjF1+4p1Q0TFxwpf9QVFNjCZ84EnGkie609NhFB8tU9k5Vkoqw3jihdsoJEUy6GK0H30dl/7H1rGxsx6Ec05aaFhiPw6t0jLxF1kj4uIUUtyjzEBEpi1D8Jgw5ToAzE/dLLvgPlMu+gZu6pyyoQxnJ/0SYRd63N82MLWgTkbP8yzNSL5FFaqjZtE5VjvjZin4GWyDB9279EZ6D6avFW2l7WuMJG++xBqGsNKZUgNM4WkUGNfCd+m42hJgt46eOy89db672su0n24IZG9tAsgQl8vPsVKfsTvTWlMj6/jISm7Dcctr1rZpSb8hRPsQstlfqMw3q6qijtTkFiMsdGRwJ6LNukSFUxOarhVsfREQngJufm4IxFpJJMR5F1DFSDPiOPuylEqXzke+j078Y4vr+QRo07YRlsoEv4a6ChcxMd3uu5Oami+D747/YVaS8kLd/3bO+WFpubID5fv4F7/JO4Fy/O7n1waPpNnzi/PZRlHVlwzNVAs09OmTmgzNM4/jAJBO9lRgCFA1SW0BADAG+OYsDBncznvyv8yKo/oM0S2E69D1NQ9l9E8fNGwNgQNP+1l2h5pLgBVt3LKl6gEII2VyxbcDkGJMFYcvk8QhPPFNzYqZw3swyXzQ3nvZqWU2ARuzo1BgMrvnDgW1H+AMbKbNGU7IYXIYaLfTR9S7qrUbHESHac4wo9J9HmRiU1/IQdyr5LldYkzYtZOrjM4SzBkYYVtpSH7Sopij/TTy0U9CXNle7iCnZQS/72C8kwyJ+BGqpULLkSWhHoj+U9GSW9UgDHZ62jRTzvuZz5QaX/hYOmpNChNMFS1zoDYVE7ZIzVQKX03IDkzHAVJCXggwhQO3NK6OGhlP7A/heM6zgiR3/LlOa8uW4fcow50XC3280SDziK0Uczab3zlYXPPH6KqGPJfnftgwuvcHsRgddOWDVfEH3Q9mAj0y1R1Fop+NI/+Rbg+iznL7GXHPnFLngtJb9NbC+hwyed49ABOTzAwt4Y2rXZ3XDuDalHoWlr1oQ1bbZK7/b1zuecVAElIQ2qDXacvJQHRIiBHfPZ3G52Z2lTf6OGg/elBurqGhA2wdDAQrBIWJwiTClONbV+8yR/4Md7aPi44E4XICLpHhE5hzko7ePy9cwh3oXy3btBt0urRwrl4d/jhHvoYt1eE2inNWEOYdlkXFUDlDErwOpFVsyQon0G25zNLAcVaZgdJLWueU1y3G0XkfHRqMZ8eV1iNAegPCCNRCvJ6SVsSwcQ67s45a8VqFxSSW0F65bDCI6Ue3Hwpb1RFKbfSIJbPyUrVSq5K99wUJ01O93Kn8LQlrAbjHWo5Za+tW0a/+Qlbr5E2eSEge+ldnbMbA9rcJwZf4bT457dBXMdlD7mECIDZtD8M/KLeyzMEinDzPfqnwZjU2ifxs6gaJPXOQAWPzbCm/z2vGlRbXDGZF6yTbLTdjzviuPhVtb7bzsZW2AYC+TlZqb4qm9MAVsH5rX3OZmvvmw5oRKeSj+FFD7uSRwfutDGC99i93uptU8syL/8Tr8xU3atxITlSqHqG+rVGWdLO9i3iq38zXgXbvZacrc3CMF5QBIM8yZXNslXH5k39D5SqubSHBWTqAJ1I0heOjaIHQGLROBYLn178tckBxfKQ2UpyfkvMw1Waw+fp5f64Ce+5bmYyZr6Dhmw/xcoAihjUsEqoecrLuGPp6qI4hQt9qOnVrAxHzwwtJGxcqoiCbe1mgz0fxMCt/i0z3ygdqAn20DKPHuBdqgVUFwx2T7Ac9fUCf3RHMq34onrr2nLHc038GYedmlFjoUZStujGwA8tSwLWyuWZTDVV+ZaW92qkhmrACog6NwhR6SEjQgsMRCVBQZzYirZxyulYmcNWH6BUmnLLFsn3GbS40xUr70gujEPnjZUK/ExGRfUPOfrYYb8mAciE9nP8OeK/UI+zjJy6Qp8mMroFw7gVHCfDtKTeQFt4JV3zubGsD7jypquHKCqPewhgn9tZ1UIsKIQB7+hBwDHzhlOZ2FfR4eLwQkO8sz275tpjHDAqX/TBWWRVg/yBDii0CWN4bP8UuX36jZKZboJUxIkM1xThiGZM2/oMbe5cZyjgrBR3P21wiDHAAlsHkaMfJgkVLqvZOw8hflKRIMa2dEYo5voD6aV30sATHQLoV0o+MlV3WA38RA+23Jqt1g+UZ7ReAuDP88jXhqWFcIvWHrJG0oy+rpAPQU/38vhIxbl//lirsirdVK2LrU47CC1f9/pRi07vTnvAm+n02dhwriqpwOmI2o2OU4mO0q96pCueKjAttkXgz+NSIJzcwprvNyE9UtKWswmIQg=", - "hash": "1104432843216666147630995603505954039049338382620925003943517086518984015452" + "data": "AAClzwJllhsXW4UHH6dHRfysOr2Bv96SCGeRT4Wa3IseIaKiigrgpOaDXTEEjlaJI+fiakjF1+4p1Q0TFxwpf9QVFNjCZ84EnGkie609NhFB8tU9k5Vkoqw3jihdsoJEUy6GK0H30dl/7H1rGxsx6Ec05aaFhiPw6t0jLxF1kj4uIUUtyjzEBEpi1D8Jgw5ToAzE/dLLvgPlMu+gZu6pyyoQxnJ/0SYRd63N82MLWgTkbP8yzNSL5FFaqjZtE5VjvjZin4GWyDB9279EZ6D6avFW2l7WuMJG++xBqGsNKZUgNM4WkUGNfCd+m42hJgt46eOy89db672su0n24IZG9tAsgQl8vPsVKfsTvTWlMj6/jISm7Dcctr1rZpSb8hRPsQstlfqMw3q6qijtTkFiMsdGRwJ6LNukSFUxOarhVsfREQngJufm4IxFpJJMR5F1DFSDPiOPuylEqXzke+j078Y4vr+QRo07YRlsoEv4a6ChcxMd3uu5Oami+D747/YVaS8kLd/3bO+WFpubID5fv4F7/JO4Fy/O7n1waPpNnzi/PZRlHVlwzNVAs09OmTmgzNM4/jAJBO9lRgCFA1SW0BADAN5EPaisxUZTo01WdepOTTqske7XVzf6CqguwBI8KIo5/ZPU30NcTGkcuZj8BeuwomRi/lbeQSoipFAgSBSEBxzPFNzYqZw3swyXzQ3nvZqWU2ARuzo1BgMrvnDgW1H+AMbKbNGU7IYXIYaLfTR9S7qrUbHESHac4wo9J9HmRiU1/IQdyr5LldYkzYtZOrjM4SzBkYYVtpSH7Sopij/TTy0U9CXNle7iCnZQS/72C8kwyJ+BGqpULLkSWhHoj+U9GSW9UgDHZ62jRTzvuZz5QaX/hYOmpNChNMFS1zoDYVE7ZIzVQKX03IDkzHAVJCXggwhQO3NK6OGhlP7A/heM6zgiR3/LlOa8uW4fcow50XC3280SDziK0Uczab3zlYXPPH6KqGPJfnftgwuvcHsRgddOWDVfEH3Q9mAj0y1R1FopvcVANaoUdvCZAXJmv0Yk34Md7qsT8/WQEemNPnewbC7pigOwNRT6y1J4KRBNKbT+ywXePMGWl4gg5uvakjjcGw2qDXacvJQHRIiBHfPZ3G52Z2lTf6OGg/elBurqGhA2wdDAQrBIWJwiTClONbV+8yR/4Md7aPi44E4XICLpHhE5hzko7ePy9cwh3oXy3btBt0urRwrl4d/jhHvoYt1eE2inNWEOYdlkXFUDlDErwOpFVsyQon0G25zNLAcVaZgdJLWueU1y3G0XkfHRqMZ8eV1iNAegPCCNRCvJ6SVsSwcQ67s45a8VqFxSSW0F65bDCI6Ue3Hwpb1RFKbfSIJbPyUrVSq5K99wUJ01O93Kn8LQlrAbjHWo5Za+tW0a/+Qlbr5E2eSEge+ldnbMbA9rcJwZf4bT457dBXMdlD7mECIDZtD8M/KLeyzMEinDzPfqnwZjU2ifxs6gaJPXOQAWPzbCm/z2vGlRbXDGZF6yTbLTdjzviuPhVtb7bzsZW2AYC+TlZqb4qm9MAVsH5rX3OZmvvmw5oRKeSj+FFD7uSRwfutDGC99i93uptU8syL/8Tr8xU3atxITlSqHqG+rVGWdLO9i3iq38zXgXbvZacrc3CMF5QBIM8yZXNslXH5k39D5SqubSHBWTqAJ1I0heOjaIHQGLROBYLn178tckBxfKQ2UpyfkvMw1Waw+fp5f64Ce+5bmYyZr6Dhmw/xcoAihjUsEqoecrLuGPp6qI4hQt9qOnVrAxHzwwtJGxcqoiCbe1mgz0fxMCt/i0z3ygdqAn20DKPHuBdqgVUFwx2T7Ac9fUCf3RHMq34onrr2nLHc038GYedmlFjoUZStujGwA8tSwLWyuWZTDVV+ZaW92qkhmrACog6NwhR6SEjQgsMRCVBQZzYirZxyulYmcNWH6BUmnLLFsn3GbS40xUr70gujEPnjZUK/ExGRfUPOfrYYb8mAciE9nP8OeK/UI+zjJy6Qp8mMroFw7gVHCfDtKTeQFt4JV3zubGsD7jypquHKCqPewhgn9tZ1UIsKIQB7+hBwDHzhlOZ2FfR4eLwQkO8sz275tpjHDAqX/TBWWRVg/yBDii0CWN4bP8UuX36jZKZboJUxIkM1xThiGZM2/oMbe5cZyjgrBR3P21wiDHAAlsHkaMfJgkVLqvZOw8hflKRIMa2dEYo5voD6aV30sATHQLoV0o+MlV3WA38RA+23Jqt1g+UZ7ReAuDP88jXhqWFcIvWHrJG0oy+rpAPQU/38vhIxbl//lirsirdVK2LrU47CC1f9/pRi07vTnvAm+n02dhwriqpwOmI2o2OU4mO0q96pCueKjAttkXgz+NSIJzcwprvNyE9UtKWswmIQg=", + "hash": "10431289529378131910378558488657048256065054528583914701305345208193612782024" } }, "sha256": { From 5da3d5d027f1968a82f67c3c20cfeb646333bd34 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 18:30:08 +0200 Subject: [PATCH 90/93] expose options including tree size --- src/index.ts | 6 +- .../actions/offchain-contract.unit-test.ts | 16 ++--- src/lib/mina/actions/offchain-state-rollup.ts | 50 +++++++------- .../actions/offchain-state-serialization.ts | 20 +++--- src/lib/mina/actions/offchain-state.ts | 65 ++++++++++++++++--- src/lib/provable/merkle-tree-indexed.ts | 2 +- 6 files changed, 108 insertions(+), 51 deletions(-) diff --git a/src/index.ts b/src/index.ts index 296b3c95f2..967223db17 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,10 @@ export { Gadgets } from './lib/provable/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; export { MerkleList, MerkleListIterator } from './lib/provable/merkle-list.js'; -import { IndexedMerkleMap } from './lib/provable/merkle-tree-indexed.js'; +import { + IndexedMerkleMap, + IndexedMerkleMapBase, +} from './lib/provable/merkle-tree-indexed.js'; export { Option } from './lib/provable/option.js'; export * as Mina from './lib/mina/mina.js'; @@ -146,6 +149,7 @@ namespace Experimental { // indexed merkle map export let IndexedMerkleMap = Experimental_.IndexedMerkleMap; + export type IndexedMerkleMap = IndexedMerkleMapBase; // offchain state export let OffchainState = OffchainState_.OffchainState; diff --git a/src/lib/mina/actions/offchain-contract.unit-test.ts b/src/lib/mina/actions/offchain-contract.unit-test.ts index 882e86efc4..1d28f79956 100644 --- a/src/lib/mina/actions/offchain-contract.unit-test.ts +++ b/src/lib/mina/actions/offchain-contract.unit-test.ts @@ -2,7 +2,6 @@ import { SmartContract, method, Mina, - State, state, PublicKey, UInt64, @@ -14,10 +13,13 @@ const proofsEnabled = true; const { OffchainState, OffchainStateCommitments } = Experimental; -const offchainState = OffchainState({ - accounts: OffchainState.Map(PublicKey, UInt64), - totalSupply: OffchainState.Field(UInt64), -}); +const offchainState = OffchainState( + { + accounts: OffchainState.Map(PublicKey, UInt64), + totalSupply: OffchainState.Field(UInt64), + }, + { logTotalCapacity: 10, maxActionsPerProof: 5 } +); class StateProof extends offchainState.Proof {} @@ -26,9 +28,7 @@ class StateProof extends offchainState.Proof {} class ExampleContract extends SmartContract { // TODO could have sugar for this like // @OffchainState.commitment offchainState = OffchainState.Commitment(); - @state(OffchainStateCommitments) offchainState = State( - OffchainStateCommitments.empty() - ); + @state(OffchainStateCommitments) offchainState = offchainState.commitments(); @method async createAccount(address: PublicKey, amountToMint: UInt64) { diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index e673de03eb..486b7a3735 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -2,7 +2,10 @@ import { Proof, ZkProgram } from '../../proof-system/zkprogram.js'; import { Bool, Field } from '../../provable/wrapped.js'; import { MerkleList, MerkleListIterator } from '../../provable/merkle-list.js'; import { Actions } from '../../../bindings/mina-transaction/transaction-leaves.js'; -import { IndexedMerkleMap } from '../../provable/merkle-tree-indexed.js'; +import { + IndexedMerkleMap, + IndexedMerkleMapBase, +} from '../../provable/merkle-tree-indexed.js'; import { Struct } from '../../provable/types/struct.js'; import { SelfProof } from '../../proof-system/zkprogram.js'; import { Provable } from '../../provable/provable.js'; @@ -12,7 +15,6 @@ import { LinearizedAction, LinearizedActionList, MerkleLeaf, - TREE_HEIGHT, updateMerkleMap, } from './offchain-state-serialization.js'; import { getProofsEnabled } from '../mina.js'; @@ -27,9 +29,6 @@ class ActionIterator extends MerkleListIterator.create( Actions.emptyActionState() ) {} -// TODO: make 31 a parameter -class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} - /** * Commitments that keep track of the current state of an offchain Merkle tree constructed from actions. * Intended to be stored on-chain. @@ -45,8 +44,8 @@ class OffchainStateCommitments extends Struct({ // actionState: ActionIterator.provable, actionState: Field, }) { - static empty() { - let emptyMerkleRoot = new IndexedMerkleMap31().root; + static emptyFromHeight(height: number) { + let emptyMerkleRoot = new (IndexedMerkleMap(height))().root; return new OffchainStateCommitments({ root: emptyMerkleRoot, actionState: Actions.emptyActionState(), @@ -61,15 +60,15 @@ class OffchainStateCommitments extends Struct({ */ function merkleUpdateBatch( { - maxActionsPerBatch, + maxActionsPerProof, maxActionsPerUpdate, }: { - maxActionsPerBatch: number; + maxActionsPerProof: number; maxActionsPerUpdate: number; }, stateA: OffchainStateCommitments, actions: ActionIterator, - tree: IndexedMerkleMap31 + tree: IndexedMerkleMapBase ): OffchainStateCommitments { // this would be unnecessary if the iterator could just be the public input actions.currentHash.assertEquals(stateA.actionState); @@ -77,7 +76,7 @@ function merkleUpdateBatch( // linearize actions into a flat MerkleList, so we don't process an insane amount of dummy actions let linearActions = LinearizedActionList.empty(); - for (let i = 0; i < maxActionsPerBatch; i++) { + for (let i = 0; i < maxActionsPerProof; i++) { let inner = actions.next().startIterating(); let isAtEnd = Bool(false); for (let i = 0; i < maxActionsPerUpdate; i++) { @@ -105,7 +104,7 @@ function merkleUpdateBatch( let intermediateTree = tree.clone(); let isValidUpdate = Bool(true); - linearActions.forEach(maxActionsPerBatch, (element, isDummy) => { + linearActions.forEach(maxActionsPerProof, (element, isDummy) => { let { action, isCheckPoint } = element; let { key, value, usesPreviousValue, previousValue } = action; @@ -145,16 +144,19 @@ function OffchainStateRollup({ * * 1967*A + 87*A*U + 2 * - * where A = maxActionsPerBatch and U = maxActionsPerUpdate. + * where A = maxActionsPerProof and U = maxActionsPerUpdate. * * To determine defaults, we set U=4 which should cover most use cases while ensuring * that the main loop which is independent of U dominates. * * Targeting ~50k constraints, to leave room for recursive verification, yields A=22. */ - maxActionsPerBatch = 22, + maxActionsPerProof = 22, maxActionsPerUpdate = 4, + logTotalCapacity = 30, } = {}) { + class IndexedMerkleMapN extends IndexedMerkleMap(logTotalCapacity + 1) {} + let offchainStateRollup = ZkProgram({ name: 'merkle-map-rollup', publicInput: OffchainStateCommitments, @@ -165,15 +167,15 @@ function OffchainStateRollup({ */ firstBatch: { // [actions, tree] - privateInputs: [ActionIterator.provable, IndexedMerkleMap31.provable], + privateInputs: [ActionIterator.provable, IndexedMerkleMapN.provable], async method( stateA: OffchainStateCommitments, actions: ActionIterator, - tree: IndexedMerkleMap31 + tree: IndexedMerkleMapN ): Promise { return merkleUpdateBatch( - { maxActionsPerBatch, maxActionsPerUpdate }, + { maxActionsPerProof: maxActionsPerProof, maxActionsPerUpdate }, stateA, actions, tree @@ -187,14 +189,14 @@ function OffchainStateRollup({ // [actions, tree, proof] privateInputs: [ ActionIterator.provable, - IndexedMerkleMap31.provable, + IndexedMerkleMapN.provable, SelfProof, ], async method( stateA: OffchainStateCommitments, actions: ActionIterator, - tree: IndexedMerkleMap31, + tree: IndexedMerkleMapN, recursiveProof: Proof< OffchainStateCommitments, OffchainStateCommitments @@ -213,7 +215,7 @@ function OffchainStateRollup({ let stateB = recursiveProof.publicOutput; return merkleUpdateBatch( - { maxActionsPerBatch, maxActionsPerUpdate }, + { maxActionsPerProof: maxActionsPerProof, maxActionsPerUpdate }, stateB, actions, tree @@ -239,10 +241,10 @@ function OffchainStateRollup({ }, async prove( - tree: IndexedMerkleMap31, + tree: IndexedMerkleMapN, actions: MerkleList> ) { - assert(tree.height === TREE_HEIGHT, 'Tree height must match'); + assert(tree.height === logTotalCapacity + 1, 'Tree height must match'); if (getProofsEnabled()) await this.compile(); // clone the tree so we don't modify the input tree = tree.clone(); @@ -281,7 +283,7 @@ function OffchainStateRollup({ } // base proof - let slice = sliceActions(iterator, maxActionsPerBatch); + let slice = sliceActions(iterator, maxActionsPerProof); let proof = await offchainStateRollup.firstBatch(inputState, slice, tree); // recursive proofs @@ -290,7 +292,7 @@ function OffchainStateRollup({ if (iterator.isAtEnd().toBoolean()) break; nProofs++; - let slice = sliceActions(iterator, maxActionsPerBatch); + let slice = sliceActions(iterator, maxActionsPerProof); proof = await offchainStateRollup.nextBatch( inputState, slice, diff --git a/src/lib/mina/actions/offchain-state-serialization.ts b/src/lib/mina/actions/offchain-state-serialization.ts index 38f435010d..2bc1df8dd9 100644 --- a/src/lib/mina/actions/offchain-state-serialization.ts +++ b/src/lib/mina/actions/offchain-state-serialization.ts @@ -25,7 +25,10 @@ import { PublicKey } from '../../provable/crypto/signature.js'; import { Provable } from '../../provable/provable.js'; import { Actions } from '../account-update.js'; import { Option } from '../../provable/option.js'; -import { IndexedMerkleMap } from '../../provable/merkle-tree-indexed.js'; +import { + IndexedMerkleMap, + IndexedMerkleMapBase, +} from '../../provable/merkle-tree-indexed.js'; export { toKeyHash, @@ -39,16 +42,11 @@ export { fetchMerkleMap, updateMerkleMap, Actionable, - TREE_HEIGHT, }; type Action = [...Field[], Field, Field]; type Actionable = ProvableHashable & ProvablePure; -// TODO: make 31 a parameter -const TREE_HEIGHT = 31; -class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} - function toKeyHash | undefined>( prefix: Field, keyType: KeyType, @@ -261,9 +259,13 @@ async function fetchMerkleLeaves( * We also deserialize a keyHash -> value map from the leaves. */ async function fetchMerkleMap( + height: number, contract: { address: PublicKey; tokenId: Field }, endActionState?: Field -): Promise<{ merkleMap: IndexedMerkleMap31; valueMap: Map }> { +): Promise<{ + merkleMap: IndexedMerkleMapBase; + valueMap: Map; +}> { let result = await Mina.fetchActions( contract.address, { endActionState }, @@ -277,7 +279,7 @@ async function fetchMerkleMap( .reverse() ); - let merkleMap = new IndexedMerkleMap31(); + let merkleMap = new (IndexedMerkleMap(height))(); let valueMap = new Map(); updateMerkleMap(leaves, merkleMap, valueMap); @@ -287,7 +289,7 @@ async function fetchMerkleMap( function updateMerkleMap( updates: MerkleLeaf[][], - tree: IndexedMerkleMap31, + tree: IndexedMerkleMapBase, valueMap?: Map ) { let intermediateTree = tree.clone(); diff --git a/src/lib/mina/actions/offchain-state.ts b/src/lib/mina/actions/offchain-state.ts index dca6093ade..75c21e9d9c 100644 --- a/src/lib/mina/actions/offchain-state.ts +++ b/src/lib/mina/actions/offchain-state.ts @@ -1,7 +1,6 @@ import { InferProvable } from '../../provable/types/struct.js'; import { Actionable, - TREE_HEIGHT, fetchMerkleLeaves, fetchMerkleMap, fromActionWithoutHashes, @@ -93,14 +92,17 @@ type OffchainState = { settle( proof: Proof ): Promise; + + /** + * Commitments to the offchain state, to use in your onchain state. + */ + commitments(): State; }; type OffchainStateContract = SmartContract & { offchainState: State; }; -class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} - /** * Offchain state for a `SmartContract`. * @@ -132,11 +134,49 @@ class IndexedMerkleMap31 extends IndexedMerkleMap(TREE_HEIGHT) {} */ function OffchainState< const Config extends { [key: string]: OffchainStateKind } ->(config: Config): OffchainState { +>( + config: Config, + options?: { + /** + * The base-2 logarithm of the total capacity of the offchain state. + * + * Example: if you want to have 1 million individual state fields and map entries available, + * set this to 20, because 2^20 ~= 1M. + * + * The default is 30, which allows for ~1 billion entries. + * + * Passing in lower numbers will reduce the number of constraints required to prove offchain state updates, + * which we will make proof creation slightly faster. + * Instead, you could also use a smaller total capacity to increase the `maxActionsPerProof`, so that fewer proofs are required, + * which will reduce the proof time even more, but only in the case of many actions. + */ + logTotalCapacity?: number; + /** + * The maximum number of offchain state actions that can be included in a single account update. + * + * In other words, you must not call `.update()` or `.overwrite()` more than this number of times in any of your smart contract methods. + * + * The default is 4. + * + * Note: When increasing this, consider decreasing `maxActionsPerProof` or `logTotalCapacity` in order to not exceed the circuit size limit. + */ + maxActionsPerUpdate?: number; + maxActionsPerProof?: number; + } +): OffchainState { + // read options + let { + logTotalCapacity = 30, + maxActionsPerUpdate = 4, + maxActionsPerProof, + } = options ?? {}; + const height = logTotalCapacity + 1; + class IndexedMerkleMapN extends IndexedMerkleMap(height) {} + // setup internal state of this "class" let internal = { _contract: undefined as OffchainStateContract | undefined, - _merkleMap: undefined as IndexedMerkleMap31 | undefined, + _merkleMap: undefined as IndexedMerkleMapN | undefined, _valueMap: undefined as Map | undefined, get contract() { @@ -160,6 +200,7 @@ function OffchainState< } let actionState = await onchainActionState(); let { merkleMap, valueMap } = await fetchMerkleMap( + height, internal.contract, actionState ); @@ -168,7 +209,11 @@ function OffchainState< return { merkleMap, valueMap }; }; - let rollup = OffchainStateRollup(); + let rollup = OffchainStateRollup({ + logTotalCapacity, + maxActionsPerProof, + maxActionsPerUpdate, + }); function contract() { let ctx = smartContractContext.get(); @@ -192,7 +237,7 @@ function OffchainState< // witness the merkle map & anchor against the onchain root let map = await Provable.witnessAsync( - IndexedMerkleMap31.provable, + IndexedMerkleMapN.provable, async () => (await merkleMaps()).merkleMap ); map.root.assertEquals(stateRoot, 'root mismatch'); @@ -340,7 +385,7 @@ function OffchainState< // - take new tree from `result` // - update value map in `prove()`, or separately based on `actions` let { merkleMap: newMerkleMap, valueMap: newValueMap } = - await fetchMerkleMap(internal.contract); + await fetchMerkleMap(height, internal.contract); internal._merkleMap = newMerkleMap; internal._valueMap = newValueMap; @@ -374,6 +419,10 @@ function OffchainState< : map(i, kind.keyType, kind.valueType), ]) ) as any, + + commitments() { + return State(OffchainStateCommitments.emptyFromHeight(height)); + }, }; } diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index f65f30ed71..262e788dfc 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -11,7 +11,7 @@ import { conditionalSwap } from './merkle-tree.js'; import { provableFromClass } from './types/provable-derivers.js'; // external API -export { IndexedMerkleMap }; +export { IndexedMerkleMap, IndexedMerkleMapBase }; // internal API export { Leaf }; From 49a793a22c48ee75447d386fa68d5f87bf326622 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 18:42:18 +0200 Subject: [PATCH 91/93] simplify offchain state commitments import --- src/lib/mina/actions/offchain-contract.unit-test.ts | 6 ++---- src/lib/mina/actions/offchain-state-rollup.ts | 6 +++--- src/lib/mina/actions/offchain-state.ts | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/lib/mina/actions/offchain-contract.unit-test.ts b/src/lib/mina/actions/offchain-contract.unit-test.ts index 1d28f79956..0398a14ee9 100644 --- a/src/lib/mina/actions/offchain-contract.unit-test.ts +++ b/src/lib/mina/actions/offchain-contract.unit-test.ts @@ -11,7 +11,7 @@ import assert from 'assert'; const proofsEnabled = true; -const { OffchainState, OffchainStateCommitments } = Experimental; +const { OffchainState } = Experimental; const offchainState = OffchainState( { @@ -26,9 +26,7 @@ class StateProof extends offchainState.Proof {} // example contract that interacts with offchain state class ExampleContract extends SmartContract { - // TODO could have sugar for this like - // @OffchainState.commitment offchainState = OffchainState.Commitment(); - @state(OffchainStateCommitments) offchainState = offchainState.commitments(); + @state(OffchainState.Commitments) offchainState = offchainState.commitments(); @method async createAccount(address: PublicKey, amountToMint: UInt64) { diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index 486b7a3735..46e374991f 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -98,7 +98,7 @@ function merkleUpdateBatch( actions.assertAtEnd(); // tree must match the public Merkle root; the method operates on the tree internally - // TODO: this would be simpler if the tree was the public input direcetly + // TODO: this would be simpler if the tree was the public input directly stateA.root.assertEquals(tree.root); let intermediateTree = tree.clone(); @@ -175,7 +175,7 @@ function OffchainStateRollup({ tree: IndexedMerkleMapN ): Promise { return merkleUpdateBatch( - { maxActionsPerProof: maxActionsPerProof, maxActionsPerUpdate }, + { maxActionsPerProof, maxActionsPerUpdate }, stateA, actions, tree @@ -215,7 +215,7 @@ function OffchainStateRollup({ let stateB = recursiveProof.publicOutput; return merkleUpdateBatch( - { maxActionsPerProof: maxActionsPerProof, maxActionsPerUpdate }, + { maxActionsPerProof, maxActionsPerUpdate }, stateB, actions, tree diff --git a/src/lib/mina/actions/offchain-state.ts b/src/lib/mina/actions/offchain-state.ts index 75c21e9d9c..18a5064415 100644 --- a/src/lib/mina/actions/offchain-state.ts +++ b/src/lib/mina/actions/offchain-state.ts @@ -428,6 +428,7 @@ function OffchainState< OffchainState.Map = OffchainMap; OffchainState.Field = OffchainField; +OffchainState.Commitments = OffchainStateCommitments; // type helpers From 84c8583f5b12c7b9e6844a51e8976806baca2886 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 13:59:04 +0200 Subject: [PATCH 92/93] fix clone() --- src/lib/provable/merkle-tree-indexed.ts | 32 +++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/lib/provable/merkle-tree-indexed.ts b/src/lib/provable/merkle-tree-indexed.ts index 1c1999857e..484ef3393b 100644 --- a/src/lib/provable/merkle-tree-indexed.ts +++ b/src/lib/provable/merkle-tree-indexed.ts @@ -102,19 +102,6 @@ class IndexedMerkleMapBase { readonly sortedLeaves: StoredLeaf[]; }>; - clone() { - let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); - cloned.root = this.root; - cloned.length = this.length; - cloned.data.updateAsProver(({ nodes, sortedLeaves }) => { - return { - nodes: nodes.map((row) => [...row]), - sortedLeaves: [...sortedLeaves], - }; - }); - return cloned; - } - // we'd like to do `abstract static provable` here but that's not supported static provable: Provable< IndexedMerkleMapBase, @@ -150,6 +137,25 @@ class IndexedMerkleMapBase { index: 0, }; + /** + * Clone the entire Merkle map. + * + * This method is provable. + */ + clone() { + let cloned = new (this.constructor as typeof IndexedMerkleMapBase)(); + cloned.root = this.root; + cloned.length = this.length; + cloned.data.updateAsProver(() => { + let { nodes, sortedLeaves } = this.data.get(); + return { + nodes: nodes.map((row) => [...row]), + sortedLeaves: [...sortedLeaves], + }; + }); + return cloned; + } + /** * Insert a new leaf `(key, value)`. * From 1ab65bb848bf4a1f7b810f6808c6d71540d8158a Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 4 Jun 2024 19:17:57 +0200 Subject: [PATCH 93/93] fix multiple proofs case --- src/lib/mina/actions/offchain-state-rollup.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/mina/actions/offchain-state-rollup.ts b/src/lib/mina/actions/offchain-state-rollup.ts index 46e374991f..c64e0a789d 100644 --- a/src/lib/mina/actions/offchain-state-rollup.ts +++ b/src/lib/mina/actions/offchain-state-rollup.ts @@ -286,6 +286,11 @@ function OffchainStateRollup({ let slice = sliceActions(iterator, maxActionsPerProof); let proof = await offchainStateRollup.firstBatch(inputState, slice, tree); + // update tree root/length again, they aren't mutated :( + // TODO: this shows why the full tree should be the public output + tree.root = proof.publicOutput.root; + tree.length = Field(tree.data.get().sortedLeaves.length); + // recursive proofs let nProofs = 1; for (let i = 1; ; i++) { @@ -299,6 +304,10 @@ function OffchainStateRollup({ tree, proof ); + + // update tree root/length again, they aren't mutated :( + tree.root = proof.publicOutput.root; + tree.length = Field(tree.data.get().sortedLeaves.length); } return { proof, tree, nProofs };