From 7664a4d4e8e5d4781191e798f2ced95b50f0629c Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Wed, 28 Jun 2023 15:35:39 -0400 Subject: [PATCH 1/3] refactor: remove `rebuildShares` --- codecs.go | 16 ++++- extendeddatacrossword.go | 46 ++++++-------- extendeddatacrossword_test.go | 110 +++++++++++++++------------------- leopard.go | 13 +++- 4 files changed, 90 insertions(+), 95 deletions(-) diff --git a/codecs.go b/codecs.go index 30f5531..ccf49ff 100644 --- a/codecs.go +++ b/codecs.go @@ -1,6 +1,9 @@ package rsmt2d -import "fmt" +import ( + "errors" + "fmt" +) const ( // Leopard is a codec that was originally implemented in the C++ library @@ -15,8 +18,11 @@ type Codec interface { // Encode encodes original data, automatically extracting share size. // There must be no missing shares. Only returns parity shares. Encode(data [][]byte) ([][]byte, error) - // Decode decodes sparse original + parity data, automatically extracting share size. - // Missing shares must be nil. Returns original + parity data. + // Decode attempts to reconstruct the missing shards in data. The data + // parameter should contain all original + parity shards where missing + // shards should be `nil`. If reconstruction is successful, the original + + // parity shards are returned. Returns ErrTooFewShards if not enough non-nil + // shards exist in data to reconstruct the missing shards. Decode(data [][]byte) ([][]byte, error) // MaxChunks returns the max. number of chunks each code supports in a 2D square. MaxChunks() int @@ -33,3 +39,7 @@ func registerCodec(ct string, codec Codec) { } codecs[ct] = codec } + +// ErrTooFewShards is returned by Decode if too few shards exist in the data to +// reconstruct the `nil` shards. +var ErrTooFewShards = errors.New("too few shards given to reconstruct all the shards in data") diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index a523018..5f20059 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -136,14 +136,18 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( shares[c] = vectorData[c] } - // Attempt rebuild - rebuiltShares, isDecoded, err := eds.rebuildShares(shares) + // Attempt to rebuild the shards in this row. + rebuiltShares, err := eds.codec.Decode(shares) if err != nil { + if err == ErrTooFewShards { + // Decode was unsuccessful for this iteration but don't propagate the + // error because that would halt the progress of solveCrossword. + return false, false, nil + } + // Otherwise, Decode was unsuccessful for some other reason and we + // should propagate the error. return false, false, err } - if !isDecoded { - return false, false, nil - } // Check that rebuilt shares matches appropriate root err = eds.verifyAgainstRowRoots(rowRoots, uint(r), rebuiltShares, noShareInsertion, nil) @@ -200,14 +204,18 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } - // Attempt rebuild - rebuiltShares, isDecoded, err := eds.rebuildShares(shares) + // Attempt to rebuild the shards in this column. + rebuiltShares, err := eds.codec.Decode(shares) if err != nil { + if err == ErrTooFewShards { + // Decode was unsuccessful for this iteration but don't propagate the + // error because that would halt the progress of solveCrossword. + return false, false, nil + } + // Otherwise, Decode was unsuccessful for some other reason and we + // should propagate the error. return false, false, err } - if !isDecoded { - return false, false, nil - } // Check that rebuilt shares matches appropriate root err = eds.verifyAgainstColRoots(colRoots, uint(c), rebuiltShares, noShareInsertion, nil) @@ -241,24 +249,6 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( return true, true, nil } -// rebuildShares attempts to rebuild a row or column of shares. -// Returns -// 1. An entire row or column of shares so original + parity shares. -// 2. Whether the original shares could be decoded from the shares parameter. -// 3. [Optional] an error. -func (eds *ExtendedDataSquare) rebuildShares( - shares [][]byte, -) ([][]byte, bool, error) { - rebuiltShares, err := eds.codec.Decode(shares) - if err != nil { - // Decode was unsuccessful but don't propagate the error because that - // would halt the progress of solveCrosswordRow or solveCrosswordCol. - return nil, false, nil - } - - return rebuiltShares, true, nil -} - func (eds *ExtendedDataSquare) verifyAgainstRowRoots( rowRoots [][]byte, r uint, diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index ac59495..00add48 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // PseudoFraudProof is an example fraud proof. @@ -19,70 +20,53 @@ type PseudoFraudProof struct { } func TestRepairExtendedDataSquare(t *testing.T) { - bufferSize := 64 - tests := []struct { - name string - // Size of each share, in bytes - shareSize int - codec Codec - }{ - {"leopard", bufferSize, NewLeoRSCodec()}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - name, codec, shareSize := test.name, test.codec, test.shareSize - original := createTestEds(codec, shareSize) - - rowRoots := original.RowRoots() - colRoots := original.ColRoots() - - // Verify that an EDS can be repaired after the maximum amount of erasures - t.Run("MaximumErasures", func(t *testing.T) { - flattened := original.Flattened() - flattened[0], flattened[2], flattened[3] = nil, nil, nil - flattened[4], flattened[5], flattened[6], flattened[7] = nil, nil, nil, nil - flattened[8], flattened[9], flattened[10] = nil, nil, nil - flattened[12], flattened[13] = nil, nil - - // Re-import the data square. - eds, err := ImportExtendedDataSquare(flattened, codec, NewDefaultTree) - if err != nil { - t.Errorf("ImportExtendedDataSquare failed: %v", err) - } - - err = eds.Repair(rowRoots, colRoots) - if err != nil { - t.Errorf("unexpected err while repairing data square: %v, codec: :%s", err, name) - } else { - assert.Equal(t, original.GetCell(0, 0), bytes.Repeat([]byte{1}, shareSize)) - assert.Equal(t, original.GetCell(0, 1), bytes.Repeat([]byte{2}, shareSize)) - assert.Equal(t, original.GetCell(1, 0), bytes.Repeat([]byte{3}, shareSize)) - assert.Equal(t, original.GetCell(1, 1), bytes.Repeat([]byte{4}, shareSize)) - } - }) - - // Verify that an EDS returns an error when there are too many erasures - t.Run("Unrepairable", func(t *testing.T) { - flattened := original.Flattened() - flattened[0], flattened[2], flattened[3] = nil, nil, nil - flattened[4], flattened[5], flattened[6], flattened[7] = nil, nil, nil, nil - flattened[8], flattened[9], flattened[10] = nil, nil, nil - flattened[12], flattened[13], flattened[14] = nil, nil, nil - - // Re-import the data square. - eds, err := ImportExtendedDataSquare(flattened, codec, NewDefaultTree) - if err != nil { - t.Errorf("ImportExtendedDataSquare failed: %v", err) - } + codec := NewLeoRSCodec() + shareSize := 64 - err = eds.Repair(rowRoots, colRoots) - if err != ErrUnrepairableDataSquare { - t.Errorf("did not return an error on trying to repair an unrepairable square") - } - }) - }) - } + // Verify that an EDS can be repaired after the maximum amount of erasures + t.Run("MaximumErasures", func(t *testing.T) { + original := createTestEds(codec, shareSize) + rowRoots := original.RowRoots() + colRoots := original.ColRoots() + + flattened := original.Flattened() + flattened[0], flattened[2], flattened[3] = nil, nil, nil + flattened[4], flattened[5], flattened[6], flattened[7] = nil, nil, nil, nil + flattened[8], flattened[9], flattened[10] = nil, nil, nil + flattened[12], flattened[13] = nil, nil + + // Re-import the data square. + eds, err := ImportExtendedDataSquare(flattened, codec, NewDefaultTree) + require.NoError(t, err) + + err = eds.Repair(rowRoots, colRoots) + require.NoError(t, err) + + assert.Equal(t, original.GetCell(0, 0), bytes.Repeat([]byte{1}, shareSize)) + assert.Equal(t, original.GetCell(0, 1), bytes.Repeat([]byte{2}, shareSize)) + assert.Equal(t, original.GetCell(1, 0), bytes.Repeat([]byte{3}, shareSize)) + assert.Equal(t, original.GetCell(1, 1), bytes.Repeat([]byte{4}, shareSize)) + }) + + // Verify that an EDS returns an error when there are too many erasures + t.Run("Unrepairable", func(t *testing.T) { + original := createTestEds(codec, shareSize) + rowRoots := original.RowRoots() + colRoots := original.ColRoots() + + flattened := original.Flattened() + flattened[0], flattened[2], flattened[3] = nil, nil, nil + flattened[4], flattened[5], flattened[6], flattened[7] = nil, nil, nil, nil + flattened[8], flattened[9], flattened[10] = nil, nil, nil + flattened[12], flattened[13], flattened[14] = nil, nil, nil + + // Re-import the data square. + eds, err := ImportExtendedDataSquare(flattened, codec, NewDefaultTree) + require.NoError(t, err) + + err = eds.Repair(rowRoots, colRoots) + assert.ErrorAs(t, err, &ErrUnrepairableDataSquare) + }) } func TestValidFraudProof(t *testing.T) { diff --git a/leopard.go b/leopard.go index cc4138f..cfe7d57 100644 --- a/leopard.go +++ b/leopard.go @@ -43,6 +43,11 @@ func (l *leoRSCodec) Encode(data [][]byte) ([][]byte, error) { return shards[dataLen:], nil } +// Decode attempts to reconstruct the missing shards in data. The data +// parameter should contain all original + parity shards where missing +// shards should be `nil`. If reconstruction is successful, the original + +// parity shards are returned. Returns ErrTooFewShards if not enough non-nil +// shards exist in data to reconstruct the missing shards. func (l *leoRSCodec) Decode(data [][]byte) ([][]byte, error) { half := len(data) / 2 enc, err := l.loadOrInitEncoder(half) @@ -50,7 +55,13 @@ func (l *leoRSCodec) Decode(data [][]byte) ([][]byte, error) { return nil, err } err = enc.Reconstruct(data) - return data, err + if err == reedsolomon.ErrTooFewShards || err == reedsolomon.ErrShardNoData { + return nil, ErrTooFewShards + } + if err != nil { + return nil, err + } + return data, nil } func (l *leoRSCodec) loadOrInitEncoder(dataLen int) (reedsolomon.Encoder, error) { From 873cf274d992df558be5127ed0ffe69c79278cc6 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Thu, 29 Jun 2023 10:25:39 -0400 Subject: [PATCH 2/3] refactor: reintroduce rebuildShares --- extendeddatacrossword.go | 52 ++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 5f20059..5fb86f1 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -118,7 +118,7 @@ func (eds *ExtendedDataSquare) solveCrossword( // Returns // - if the row is solved (i.e. complete) // - if the row was previously unsolved and now solved -// - an error if the repair is unsuccessful +// - an error if the repair is unsuccessful and solveCrossword should halt func (eds *ExtendedDataSquare) solveCrosswordRow( r int, rowRoots [][]byte, @@ -136,18 +136,13 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( shares[c] = vectorData[c] } - // Attempt to rebuild the shards in this row. - rebuiltShares, err := eds.codec.Decode(shares) + rebuiltShares, isRebuilt, err := eds.rebuildShares(shares) if err != nil { - if err == ErrTooFewShards { - // Decode was unsuccessful for this iteration but don't propagate the - // error because that would halt the progress of solveCrossword. - return false, false, nil - } - // Otherwise, Decode was unsuccessful for some other reason and we - // should propagate the error. return false, false, err } + if !isRebuilt { + return false, false, nil + } // Check that rebuilt shares matches appropriate root err = eds.verifyAgainstRowRoots(rowRoots, uint(r), rebuiltShares, noShareInsertion, nil) @@ -185,7 +180,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( // Returns // - if the column is solved (i.e. complete) // - if the column was previously unsolved and now solved -// - an error if the repair is unsuccessful +// - an error if the repair is unsuccessful and solveCrossword should halt. func (eds *ExtendedDataSquare) solveCrosswordCol( c int, rowRoots [][]byte, @@ -204,18 +199,13 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } - // Attempt to rebuild the shards in this column. - rebuiltShares, err := eds.codec.Decode(shares) + rebuiltShares, isRebuilt, err := eds.rebuildShares(shares) if err != nil { - if err == ErrTooFewShards { - // Decode was unsuccessful for this iteration but don't propagate the - // error because that would halt the progress of solveCrossword. - return false, false, nil - } - // Otherwise, Decode was unsuccessful for some other reason and we - // should propagate the error. return false, false, err } + if !isRebuilt { + return false, false, nil + } // Check that rebuilt shares matches appropriate root err = eds.verifyAgainstColRoots(colRoots, uint(c), rebuiltShares, noShareInsertion, nil) @@ -249,6 +239,28 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( return true, true, nil } +// rebuildShares attempts to rebuild a row or column of shares. +// The missing shares should be set to nil. +// Returns +// 1. All rebuilt shares (so original + parity shares). +// 2. Whether the shares were rebuilt successfully. +// 3. [Optional] an error. +func (eds *ExtendedDataSquare) rebuildShares(shares [][]byte) ([][]byte, bool, error) { + rebuiltShares, err := eds.codec.Decode(shares) + if err != nil { + if err == ErrTooFewShards { + // Decode was unsuccessful for this iteration but don't propagate the + // error because that would halt the progress of solveCrossword. + return [][]byte{}, false, nil + } + // Otherwise, Decode was unsuccessful for some other reason and we + // should propagate the error. + return [][]byte{}, false, err + } + + return rebuiltShares, true, nil +} + func (eds *ExtendedDataSquare) verifyAgainstRowRoots( rowRoots [][]byte, r uint, From 81d1bfffded3e136ad8480b0903eeb32424c2b68 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Thu, 29 Jun 2023 10:29:35 -0400 Subject: [PATCH 3/3] remove trailing period --- extendeddatacrossword.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 5fb86f1..f4d52c9 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -180,7 +180,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( // Returns // - if the column is solved (i.e. complete) // - if the column was previously unsolved and now solved -// - an error if the repair is unsuccessful and solveCrossword should halt. +// - an error if the repair is unsuccessful and solveCrossword should halt func (eds *ExtendedDataSquare) solveCrosswordCol( c int, rowRoots [][]byte,