diff --git a/erigon-lib/commitment/commitment.go b/erigon-lib/commitment/commitment.go index 1e60c1f790f..ad6a5d183dd 100644 --- a/erigon-lib/commitment/commitment.go +++ b/erigon-lib/commitment/commitment.go @@ -138,7 +138,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * default: trie := NewHexPatriciaHashed(length.Addr, nil, tmpdir) - tree := NewUpdates(mode, tmpdir, trie.HashAndNibblizeKey) + tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) return trie, tree } } diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index 3d692dca755..561d276df68 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -342,31 +342,6 @@ func (cell *cell) fillFromLowerCell(lowCell *cell, lowDepth int, preExtension [] cell.loaded = lowCell.loaded } -func hashKey(keccak keccakState, plainKey []byte, dest []byte, hashedKeyOffset int, hashBuf []byte) error { - _, _ = hashBuf[length.Hash-1], dest[length.Hash*2-1] // bounds checks elimination - keccak.Reset() - if _, err := keccak.Write(plainKey); err != nil { - return err - } - if _, err := keccak.Read(hashBuf); err != nil { - return err - } - hashBuf = hashBuf[hashedKeyOffset/2:] - var k int - if hashedKeyOffset%2 == 1 { - dest[0] = hashBuf[0] & 0xf - k++ - hashBuf = hashBuf[1:] - } - for _, c := range hashBuf { - dest[k] = (c >> 4) & 0xf - k++ - dest[k] = c & 0xf - k++ - } - return nil -} - func (cell *cell) deriveHashedKeys(depth int, keccak keccakState, accountKeyLen int) error { extraLen := 0 if cell.accountAddrLen > 0 { @@ -630,7 +605,7 @@ func (hph *HexPatriciaHashed) accountLeafHashWithKey(buf, key []byte, val rlp.Rl } else { compactLen = len(key)/2 + 1 if len(key)&1 == 1 { - compact0 = 16 + key[0] // Odd (1<<4) + first nibble + compact0 = terminatorHexByte + key[0] // Odd (1<<4) + first nibble ni = 1 } } @@ -749,7 +724,7 @@ func (hph *HexPatriciaHashed) computeCellHashWithStorage(cell *cell, depth int, if err = hashKey(hph.keccak, cell.storageAddr[koffset:cell.storageAddrLen], cell.hashedExtension[:], hashedKeyOffset, cell.hashBuf[:]); err != nil { return nil, storageRootHashIsSet, nil, err } - cell.hashedExtension[64-hashedKeyOffset] = 16 // Add terminator + cell.hashedExtension[64-hashedKeyOffset] = terminatorHexByte // Add terminator if cell.stateHashLen > 0 { res := append([]byte{160}, cell.stateHash[:cell.stateHashLen]...) @@ -814,7 +789,7 @@ func (hph *HexPatriciaHashed) computeCellHashWithStorage(cell *cell, depth int, if err := hashKey(hph.keccak, cell.accountAddr[:cell.accountAddrLen], cell.hashedExtension[:], depth, cell.hashBuf[:]); err != nil { return nil, storageRootHashIsSet, nil, err } - cell.hashedExtension[64-depth] = 16 // Add terminator + cell.hashedExtension[64-depth] = terminatorHexByte // Add terminator if !storageRootHashIsSet { if cell.extLen > 0 { // Extension if cell.hashLen == 0 { @@ -919,7 +894,7 @@ func (hph *HexPatriciaHashed) computeCellHash(cell *cell, depth int, buf []byte) if err = cell.hashStorageKey(hph.keccak, koffset, 0, hashedKeyOffset); err != nil { return nil, err } - cell.hashedExtension[64-hashedKeyOffset] = 16 // Add terminator + cell.hashedExtension[64-hashedKeyOffset] = terminatorHexByte // Add terminator if cell.stateHashLen > 0 { hph.keccak.Reset() @@ -966,7 +941,7 @@ func (hph *HexPatriciaHashed) computeCellHash(cell *cell, depth int, buf []byte) if err := cell.hashAccKey(hph.keccak, depth); err != nil { return nil, err } - cell.hashedExtension[64-depth] = 16 // Add terminator + cell.hashedExtension[64-depth] = terminatorHexByte // Add terminator if !storageRootHashIsSet { if cell.extLen > 0 { // Extension if cell.hashLen == 0 { @@ -1231,7 +1206,7 @@ func (hph *HexPatriciaHashed) ToTrie(hashedKey []byte, codeReads map[libcommon.H extensionKey := make([]byte, extKeyLength) copy(extensionKey, hashedExtKey) if keyPos+1 == len(hashedKey) || keyPos+1 == 64 { - extensionKey[len(extensionKey)-1] = 16 // append terminator byte + extensionKey[len(extensionKey)-1] = terminatorHexByte // append terminator byte } nextNode = &trie.ShortNode{Key: extensionKey} // Value will be in the next iteration if keyPos+1 == len(hashedKey) { @@ -1329,7 +1304,7 @@ func (hph *HexPatriciaHashed) ToTrie(hashedKey []byte, codeReads map[libcommon.H // unfoldBranchNode returns true if unfolding has been done func (hph *HexPatriciaHashed) unfoldBranchNode(row, depth int, deleted bool) (bool, error) { - key := hexToCompact(hph.currentKey[:hph.currentKeyLen]) + key := hexNibblesToCompactBytes(hph.currentKey[:hph.currentKeyLen]) branchData, fileEndTxNum, err := hph.ctx.Branch(key) if err != nil { return false, err @@ -1339,7 +1314,8 @@ func (hph *HexPatriciaHashed) unfoldBranchNode(row, depth int, deleted bool) (bo branchData = branchData[2:] // skip touch map and keep the rest } if hph.trace { - fmt.Printf("unfoldBranchNode prefix '%x', nibbles [%x] depth %d row %d '%x'\n", key, hph.currentKey[:hph.currentKeyLen], depth, row, branchData) + fmt.Printf("unfoldBranchNode prefix '%x', nibbles [%x] depth %d row %d '%x'\n", + key, hph.currentKey[:hph.currentKeyLen], depth, row, branchData) } if !hph.rootChecked && hph.currentKeyLen == 0 && len(branchData) == 0 { // Special case - empty or deleted root @@ -1348,7 +1324,7 @@ func (hph *HexPatriciaHashed) unfoldBranchNode(row, depth int, deleted bool) (bo } if len(branchData) == 0 { log.Warn("got empty branch data during unfold", "key", hex.EncodeToString(key), "row", row, "depth", depth, "deleted", deleted) - return false, fmt.Errorf("empty branch data read during unfold, prefix %x", hexToCompact(hph.currentKey[:hph.currentKeyLen])) + return false, fmt.Errorf("empty branch data read during unfold, prefix %x", key) } hph.branchBefore[row] = true bitmap := binary.BigEndian.Uint16(branchData[0:]) @@ -1487,16 +1463,6 @@ type skipStat struct { accLoaded, accSkipped, accReset, storReset, storLoaded, storSkipped uint64 } -func updatedNibs(num uint16) string { - var nibbles []string - for i := 0; i < 16; i++ { - if num&(1< 0 - buf := make([]byte, bufLen) - buf[0] = zeroByte - return decodeKey(key[keyPos:], buf) -} - -func makeCompactZeroByte(key []byte) (compactZeroByte byte, keyPos, keyLen int) { - keyLen = len(key) - if hasTerm(key) { - keyLen-- - compactZeroByte = 0x20 - } - var firstNibble byte - if len(key) > 0 { - firstNibble = key[0] - } - if keyLen&1 == 1 { - compactZeroByte |= 0x10 | firstNibble // Odd: (1<<4) + first nibble - keyPos++ - } - - return -} - -func decodeKey(key, buf []byte) []byte { - keyLen := len(key) - if hasTerm(key) { - keyLen-- - } - for keyIndex, bufIndex := 0, 1; keyIndex < keyLen; keyIndex, bufIndex = keyIndex+2, bufIndex+1 { - if keyIndex == keyLen-1 { - buf[bufIndex] = buf[bufIndex] & 0x0f - } else { - buf[bufIndex] = key[keyIndex+1] - } - buf[bufIndex] |= key[keyIndex] << 4 - } - return buf -} - -func CompactedKeyToHex(compact []byte) []byte { - if len(compact) == 0 { - return compact - } - base := keybytesToHexNibbles(compact) - // delete terminator flag - if base[0] < 2 { - base = base[:len(base)-1] - } - // apply odd flag - chop := 2 - base[0]&1 - return base[chop:] -} - -func keybytesToHexNibbles(str []byte) []byte { - l := len(str)*2 + 1 - var nibbles = make([]byte, l) - for i, b := range str { - nibbles[i*2] = b / 16 - nibbles[i*2+1] = b % 16 - } - nibbles[l-1] = 16 - return nibbles -} - -// hasTerm returns whether a hex key has the terminator flag. -func hasTerm(s []byte) bool { - return len(s) > 0 && s[len(s)-1] == 16 -} - -func commonPrefixLen(b1, b2 []byte) int { - var i int - for i = 0; i < len(b1) && i < len(b2); i++ { - if b1[i] != b2[i] { - break - } - } - return i -} - -// nolint -// Hashes provided key and expands resulting hash into nibbles (each byte split into two nibbles by 4 bits) -func (hph *HexPatriciaHashed) HashAndNibblizeKey(key []byte) []byte { - hashedKey := make([]byte, length.Hash) - - hph.keccak.Reset() - fp := length.Addr - if len(key) < length.Addr { - fp = len(key) - } - hph.keccak.Write(key[:fp]) - hph.keccak.Read(hashedKey[:length.Hash]) - - if len(key[fp:]) > 0 { - hashedKey = append(hashedKey, make([]byte, length.Hash)...) - hph.keccak.Reset() - hph.keccak.Write(key[fp:]) - hph.keccak.Read(hashedKey[length.Hash:]) - } - - nibblized := make([]byte, len(hashedKey)*2) - for i, b := range hashedKey { - nibblized[i*2] = (b >> 4) & 0xf - nibblized[i*2+1] = b & 0xf - } - return nibblized -} - -func nibblize(key []byte) []byte { // nolint:unused - nibblized := make([]byte, len(key)*2) - for i, b := range key { - nibblized[i*2] = (b >> 4) & 0xf - nibblized[i*2+1] = b & 0xf - } - return nibblized -} - -// compactKey takes a slice of nibbles and compacts them into the original byte slice. -// It returns an error if the input contains invalid nibbles (values > 0xF). -func compactKey(nibbles []byte) ([]byte, error) { - // If the number of nibbles is odd, you might decide to handle it differently. - // For this example, we'll return an error. - if len(nibbles)%2 != 0 { - return nil, errors.New("nibbles slice has an odd length") - } - - key := make([]byte, len(nibbles)/2) - for i := 0; i < len(key); i++ { - highNibble := nibbles[i*2] - lowNibble := nibbles[i*2+1] - - // Validate that each nibble is indeed a nibble - if highNibble > 0xF || lowNibble > 0xF { - return nil, fmt.Errorf("invalid nibble at position %d or %d: 0x%X, 0x%X", i*2, i*2+1, highNibble, lowNibble) - } - - key[i] = (highNibble << 4) | (lowNibble & 0x0F) - } - return key, nil -} - func (hph *HexPatriciaHashed) Grid() [128][16]cell { return hph.grid } diff --git a/erigon-lib/commitment/hex_patricia_hashed_bench_test.go b/erigon-lib/commitment/hex_patricia_hashed_bench_test.go index 3db0373925c..2ea52303874 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_bench_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_bench_test.go @@ -49,7 +49,7 @@ func Benchmark_HexPatriciaHashed_Process(b *testing.B) { require.NoError(b, err) hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir()) - upds := WrapKeyUpdates(b, ModeDirect, hph.HashAndNibblizeKey, nil, nil) + upds := WrapKeyUpdates(b, ModeDirect, KeyToHexNibbleHash, nil, nil) defer upds.Close() b.ResetTimer() diff --git a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go index 76d06d2d4cd..aed644d7aeb 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go @@ -65,13 +65,13 @@ func Fuzz_ProcessUpdate(f *testing.F) { err = ms2.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - upds := WrapKeyUpdates(t, ModeDirect, hph.HashAndNibblizeKey, nil, nil) + upds := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, nil, nil) rootHashDirect, err := hph.Process(ctx, upds, "") require.NoError(t, err) require.Len(t, rootHashDirect, length.Hash, "invalid root hash length") upds.Close() - anotherUpds := WrapKeyUpdates(t, ModeUpdate, hphAnother.HashAndNibblizeKey, nil, nil) + anotherUpds := WrapKeyUpdates(t, ModeUpdate, KeyToHexNibbleHash, nil, nil) rootHashUpdate, err := hphAnother.Process(ctx, anotherUpds, "") require.NoError(t, err) require.Len(t, rootHashUpdate, length.Hash, "invalid root hash length") @@ -149,7 +149,7 @@ func Fuzz_ProcessUpdates_ArbitraryUpdateCount2(f *testing.F) { err := ms.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsDirect := WrapKeyUpdates(t, ModeDirect, hph.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsDirect := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) rootHashDirect, err := hph.Process(ctx, updsDirect, "") updsDirect.Close() require.NoError(t, err) @@ -158,7 +158,7 @@ func Fuzz_ProcessUpdates_ArbitraryUpdateCount2(f *testing.F) { err = ms2.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - upds := WrapKeyUpdates(t, ModeUpdate, hphAnother.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + upds := WrapKeyUpdates(t, ModeUpdate, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) rootHashAnother, err := hphAnother.Process(ctx, upds, "") upds.Close() require.NoError(t, err) @@ -213,7 +213,7 @@ func Fuzz_HexPatriciaHashed_ReviewKeys(f *testing.F) { t.Fatal(err) } - upds := WrapKeyUpdates(t, ModeDirect, hph.HashAndNibblizeKey, plainKeys, updates) + upds := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer upds.Close() rootHash, err := hph.Process(ctx, upds, "") diff --git a/erigon-lib/commitment/hex_patricia_hashed_test.go b/erigon-lib/commitment/hex_patricia_hashed_test.go index e2cc95bed09..8f3def470b4 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_test.go @@ -53,7 +53,7 @@ func Test_HexPatriciaHashed_ResetThenSingularUpdates(t *testing.T) { Storage("05", "04", "9898"). Build() - upds := WrapKeyUpdates(t, ModeDirect, hph.HashAndNibblizeKey, plainKeys, updates) + upds := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer upds.Close() fmt.Printf("1. Generated %d updates\n", len(updates)) @@ -121,7 +121,7 @@ func Test_HexPatriciaHashed_EmptyUpdate(t *testing.T) { err := ms.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - upds := WrapKeyUpdates(t, ModeDirect, hph.HashAndNibblizeKey, plainKeys, updates) + upds := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer upds.Close() hashBeforeEmptyUpdate, err := hph.Process(ctx, upds, "") @@ -174,7 +174,7 @@ func Test_HexPatriciaHashed_UniqueRepresentation2(t *testing.T) { err := msOne.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, trieOne.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := trieOne.Process(ctx, updsOne, "") require.NoError(t, err) @@ -189,7 +189,7 @@ func Test_HexPatriciaHashed_UniqueRepresentation2(t *testing.T) { err := msTwo.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, trieTwo.HashAndNibblizeKey, plainKeys, updates) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) fmt.Printf("\n2. Trie batch update (%d updates)\n", len(updates)) rh, err := trieTwo.Process(ctx, updsTwo, "") @@ -215,7 +215,7 @@ func Test_HexPatriciaHashed_UniqueRepresentation2(t *testing.T) { err := msOne.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, trieOne.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := trieOne.Process(ctx, updsOne, "") require.NoError(t, err) @@ -231,7 +231,7 @@ func Test_HexPatriciaHashed_UniqueRepresentation2(t *testing.T) { err := msTwo.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, trieTwo.HashAndNibblizeKey, plainKeys, updates) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) rh, err := trieTwo.Process(ctx, updsTwo, "") require.NoError(t, err) @@ -249,7 +249,7 @@ func sortUpdatesByHashIncrease(t *testing.T, hph *HexPatriciaHashed, plainKeys [ ku := make([]*KeyUpdate, len(plainKeys)) for i, pk := range plainKeys { - ku[i] = &KeyUpdate{plainKey: string(pk), hashedKey: hph.HashAndNibblizeKey(pk), update: &updates[i]} + ku[i] = &KeyUpdate{plainKey: string(pk), hashedKey: KeyToHexNibbleHash(pk), update: &updates[i]} } sort.Slice(updates, func(i, j int) bool { @@ -316,7 +316,7 @@ func Test_HexPatriciaHashed_BrokenUniqueRepr(t *testing.T) { err := stateSeq.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, trieSequential.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := trieSequential.Process(ctx, updsOne, "") require.NoError(t, err) @@ -332,7 +332,7 @@ func Test_HexPatriciaHashed_BrokenUniqueRepr(t *testing.T) { err := stateBatch.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, trieBatch.HashAndNibblizeKey, plainKeys, updates) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) rh, err := trieBatch.Process(ctx, updsTwo, "") require.NoError(t, err) @@ -399,7 +399,7 @@ func Test_HexPatriciaHashed_UniqueRepresentation(t *testing.T) { err := stateSeq.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, trieSequential.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := trieSequential.Process(ctx, updsOne, "") require.NoError(t, err) @@ -415,7 +415,7 @@ func Test_HexPatriciaHashed_UniqueRepresentation(t *testing.T) { err := stateBatch.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, trieBatch.HashAndNibblizeKey, plainKeys, updates) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) rh, err := trieBatch.Process(ctx, updsTwo, "") require.NoError(t, err) @@ -485,7 +485,7 @@ func Test_HexPatriciaHashed_Sepolia(t *testing.T) { err := state.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - upds := WrapKeyUpdates(t, ModeDirect, hph.HashAndNibblizeKey, plainKeys, updates) + upds := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) rootHash, err := hph.Process(ctx, upds, "") require.NoError(t, err) require.EqualValues(t, testData.expectedRoot, fmt.Sprintf("%x", rootHash)) @@ -598,7 +598,7 @@ func Test_HexPatriciaHashed_StateEncodeDecodeSetup(t *testing.T) { err := ms.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - upds := WrapKeyUpdates(t, ModeDirect, before.HashAndNibblizeKey, plainKeys, updates) + upds := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer upds.Close() // process updates @@ -659,7 +659,7 @@ func Test_HexPatriciaHashed_StateRestoreAndContinue(t *testing.T) { err = msTwo.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updOne := WrapKeyUpdates(t, ModeDirect, trieOne.HashAndNibblizeKey, plainKeys, updates) + updOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer updOne.Close() withoutRestore, err := trieOne.Process(ctx, updOne, "") @@ -717,7 +717,7 @@ func Test_HexPatriciaHashed_StateRestoreAndContinue(t *testing.T) { t.Logf("batch without restore (%d) root %x\n", len(updates), withoutRestore) - updTwo := WrapKeyUpdates(t, ModeDirect, trieTwo.HashAndNibblizeKey, plainKeys, updates) + updTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer updTwo.Close() afterRestore, err := trieTwo.Process(ctx, updTwo, "") @@ -758,7 +758,7 @@ func Test_HexPatriciaHashed_RestoreAndContinue(t *testing.T) { err := ms.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updTwo := WrapKeyUpdates(t, ModeDirect, trieOne.HashAndNibblizeKey, plainKeys, updates) + updTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) defer updTwo.Close() beforeRestore, err := trieTwo.Process(ctx, updTwo, "") @@ -844,7 +844,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation_AfterStateRestor err := stateSeq.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, trieSequential.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := trieSequential.Process(ctx, updsOne, "") require.NoError(t, err) @@ -871,7 +871,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation_AfterStateRestor err := stateBatch.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, trieBatch.HashAndNibblizeKey, plainKeys, updates) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) rh, err := trieBatch.Process(ctx, updsTwo, "") require.NoError(t, err) @@ -944,7 +944,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentationInTheMiddle(t *te err := stateSeq.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, sequential.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := sequential.Process(ctx, updsOne, "") require.NoError(t, err) @@ -972,7 +972,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentationInTheMiddle(t *te err := stateBatch.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, batch.HashAndNibblizeKey, plainKeys[:somewhere+1], updates[:somewhere+1]) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[:somewhere+1], updates[:somewhere+1]) rh, err := batch.Process(ctx, updsTwo, "") require.NoError(t, err) @@ -1276,7 +1276,7 @@ func Test_HexPatriciaHashed_ProcessWithDozensOfStorageKeys(t *testing.T) { //prefixesCnt := make(map[string]int) //for i := 0; i < 5000000; i++ { // rnd.Read(noise) - // //hashed := trieOne.HashAndNibblizeKey(noise) + // //hashed := trieOne.KeyToHexNibbleHash(noise) // trieOne.keccak.Reset() // trieOne.keccak.Write(noise) // hashed := make([]byte, 32) @@ -1310,7 +1310,7 @@ func Test_HexPatriciaHashed_ProcessWithDozensOfStorageKeys(t *testing.T) { err := msOne.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]) require.NoError(t, err) - updsOne := WrapKeyUpdates(t, ModeDirect, trieOne.HashAndNibblizeKey, plainKeys[i:i+1], updates[i:i+1]) + updsOne := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys[i:i+1], updates[i:i+1]) sequentialRoot, err := trieOne.Process(ctx, updsOne, "") require.NoError(t, err) @@ -1325,7 +1325,7 @@ func Test_HexPatriciaHashed_ProcessWithDozensOfStorageKeys(t *testing.T) { err := msTwo.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) - updsTwo := WrapKeyUpdates(t, ModeDirect, trieTwo.HashAndNibblizeKey, plainKeys, updates) + updsTwo := WrapKeyUpdates(t, ModeDirect, KeyToHexNibbleHash, plainKeys, updates) fmt.Printf("\n2. Trie batch update (%d updates)\n", len(updates)) rh, err := trieTwo.Process(ctx, updsTwo, "") diff --git a/erigon-lib/commitment/keys_nibbles.go b/erigon-lib/commitment/keys_nibbles.go new file mode 100644 index 00000000000..cb991184900 --- /dev/null +++ b/erigon-lib/commitment/keys_nibbles.go @@ -0,0 +1,147 @@ +package commitment + +import ( + "errors" + "fmt" + "github.com/erigontech/erigon-lib/common/length" + ecrypto "github.com/erigontech/erigon-lib/crypto" + "strings" +) + +// KeyToHexNibbleHash hashes plain key with respect to plain key size (part < 20 bytes for account, part >= 20 bytes for storage) +// and returns the hashed key in nibblized form suitable for hex trie (each byte represented by 2 nibbles). +func KeyToHexNibbleHash(key []byte) []byte { + keyLen := min(length.Addr, len(key)) + + addrHash := ecrypto.Keccak256(key[:keyLen]) + if len(key) > length.Addr { // storage + storageKeyHash := ecrypto.Keccak256(key[length.Addr:]) + addrHash = append(addrHash, storageKeyHash...) + } + return splitOntoHexNibbles(addrHash) +} + +// hexNibblesToCompactBytes Converts slice of hex nibbles into regular bytes form, combining two nibbles into one byte. +func hexNibblesToCompactBytes(key []byte) []byte { + var compactZeroByte byte + keyLen := len(key) + if hasTerm(key) { // trim terminator if needed + keyLen-- + compactZeroByte = 0x20 + } + var firstNibble byte + if len(key) > 0 { + firstNibble = key[0] + } + + // decode first byte + var keyIndex int + if keyLen&1 == 1 { // check if key length is odd + compactZeroByte |= 0x10 | firstNibble // Odd: (1<<4) + first nibble + keyIndex++ + } + + bufLen := keyLen/2 + 1 // always > 0 + buf := make([]byte, bufLen) + buf[0] = compactZeroByte + + // decode the rest of the key + for bufIndex := 1; keyIndex < keyLen; keyIndex, bufIndex = keyIndex+2, bufIndex+1 { + if keyIndex == keyLen-1 { + buf[bufIndex] = buf[bufIndex] & 0xf + } else { + buf[bufIndex] = key[keyIndex+1] + } + buf[bufIndex] |= key[keyIndex] << 4 + } + return buf +} + +// hasTerm returns whether a hex nibble key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == terminatorHexByte +} + +// returns the length of the common prefix of two byte slices +func commonPrefixLen(b1, b2 []byte) int { + var i int + for i = 0; i < len(b1) && i < len(b2); i++ { + if b1[i] != b2[i] { + break + } + } + return i +} + +// splits each byte in key slice onto 2 nibbles in the resulting slice +func splitOntoHexNibbles(key []byte) (nibblized []byte) { // nolint:unused + nibblized = make([]byte, len(key)*2) + for i, b := range key { + nibblized[i*2] = (b >> 4) & 0xf + nibblized[i*2+1] = b & 0xf + } + return nibblized +} + +// compactKey takes a slice of nibbles and compacts them into the original byte slice. +// It returns an error if the input contains invalid nibbles (values > 0xF). +func compactKey(nibbles []byte) ([]byte, error) { + // If the number of nibbles is odd, you might decide to handle it differently. + // For this example, we'll return an error. + if len(nibbles)%2 != 0 { + return nil, errors.New("nibbles slice has an odd length") + } + + key := make([]byte, len(nibbles)/2) + for i := 0; i < len(key); i++ { + highNibble := nibbles[i*2] + lowNibble := nibbles[i*2+1] + + // Validate that each nibble is indeed a nibble + if highNibble > 0xF || lowNibble > 0xF { + return nil, fmt.Errorf("invalid nibble at position %d or %d: 0x%X, 0x%X", i*2, i*2+1, highNibble, lowNibble) + } + + key[i] = (highNibble << 4) | (lowNibble & 0x0F) + } + return key, nil +} + +// updatedNibs returns a string of nibbles that are set in the given number. +func updatedNibs(num uint16) string { + var nibbles []string + for i := 0; i < 16; i++ { + if num&(1<> 4) & 0xf + k++ + dest[k] = c & 0xf + k++ + } + return nil +} diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go index 1b8f9a7f46d..4ae182a62d6 100644 --- a/turbo/jsonrpc/eth_call.go +++ b/turbo/jsonrpc/eth_call.go @@ -576,7 +576,7 @@ func (api *BaseAPI) getWitness(ctx context.Context, db kv.RoDB, blockNrOrHash rp // define these keys as "updates", but we are not really updating anything, we just want to load them into the grid, // so this is just to satisfy the current hex patricia trie api. - updates := commitment.NewUpdates(commitment.ModeDirect, sdCtx.TempDir(), hph.HashAndNibblizeKey) + updates := commitment.NewUpdates(commitment.ModeDirect, sdCtx.TempDir(), commitment.KeyToHexNibbleHash) for _, key := range touchedPlainKeys { updates.TouchPlainKey(string(key), nil, updates.TouchAccount) }