diff --git a/docker/config/prover/v3/prover-config.toml b/docker/config/prover/v3/prover-config.toml index 7f889515a..3e1653c9a 100644 --- a/docker/config/prover/v3/prover-config.toml +++ b/docker/config/prover/v3/prover-config.toml @@ -13,7 +13,7 @@ requests_root_dir = "/data/prover/v3/execution" [blob_decompression] prover_mode = "dev" requests_root_dir = "/data/prover/v3/compression" -dict_path = "/opt/linea/prover/lib/compressor/compressor_dict.bin" +dict_paths = ["/opt/linea/prover/lib/compressor/compressor_dict.bin"] [aggregation] prover_mode = "dev" diff --git a/prover/backend/aggregation/prove.go b/prover/backend/aggregation/prove.go index b437e8d98..2f6b81950 100644 --- a/prover/backend/aggregation/prove.go +++ b/prover/backend/aggregation/prove.go @@ -105,11 +105,10 @@ func makePiProof(cfg *config.Config, cf *CollectedFields) (plonk.Proof, witness. } assignment, err := c.Assign(pi_interconnection.Request{ - DictPath: cfg.BlobDecompression.DictPath, Decompressions: cf.DecompressionPI, Executions: cf.ExecutionPI, Aggregation: cf.AggregationPublicInput(cfg), - }) + }, cfg.BlobDecompressionDictStore(string(circuits.BlobDecompressionV1CircuitID))) // TODO @Tabaie: when there is a version 2, input the compressor version to use here if err != nil { return nil, nil, fmt.Errorf("could not assign the public input circuit: %w", err) } diff --git a/prover/backend/blobdecompression/prove.go b/prover/backend/blobdecompression/prove.go index 79c927268..f46e21e5f 100644 --- a/prover/backend/blobdecompression/prove.go +++ b/prover/backend/blobdecompression/prove.go @@ -4,8 +4,6 @@ import ( "bytes" "encoding/base64" "fmt" - "os" - blob_v0 "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v0" blob_v1 "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1" @@ -68,14 +66,9 @@ func Prove(cfg *config.Config, req *Request) (*Response, error) { return nil, fmt.Errorf("unsupported blob version: %v", version) } - dictPath := cfg.BlobDecompressionDictPath(string(circuitID)) - - logrus.Infof("reading the dictionary at %v", dictPath) + logrus.Info("reading dictionaries") - dict, err := os.ReadFile(dictPath) - if err != nil { - return nil, fmt.Errorf("error reading the dictionary: %w", err) - } + dictStore := cfg.BlobDecompressionDictStore(string(circuitID)) // This computes the assignment @@ -88,7 +81,7 @@ func Prove(cfg *config.Config, req *Request) (*Response, error) { assignment, pubInput, _snarkHash, err := blobdecompression.Assign( utils.RightPad(blobBytes, expectedMaxUsableBytes), - dict, + dictStore, req.Eip4844Enabled, xBytes, y, diff --git a/prover/circuits/blobdecompression/circuit.go b/prover/circuits/blobdecompression/circuit.go index 6be2bba13..0c78ce846 100644 --- a/prover/circuits/blobdecompression/circuit.go +++ b/prover/circuits/blobdecompression/circuit.go @@ -2,6 +2,7 @@ package blobdecompression import ( "errors" + "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/dictionary" "github.com/consensys/gnark-crypto/ecc/bls12-377/fr" fr381 "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" @@ -19,12 +20,12 @@ func Compile(dictionaryNbBytes int) constraint.ConstraintSystem { // Assign the circuit with concrete data. Returns the assigned circuit and the // public input computed during the assignment. -func Assign(blobData []byte, dict []byte, eip4844Enabled bool, x [32]byte, y fr381.Element) (circuit frontend.Circuit, publicInput fr.Element, snarkHash []byte, err error) { +func Assign(blobData []byte, dictStore dictionary.Store, eip4844Enabled bool, x [32]byte, y fr381.Element) (circuit frontend.Circuit, publicInput fr.Element, snarkHash []byte, err error) { switch blob.GetVersion(blobData) { case 1: - return v1.Assign(blobData, dict, eip4844Enabled, x, y) + return v1.Assign(blobData, dictStore, eip4844Enabled, x, y) case 0: - return v0.Assign(blobData, dict, eip4844Enabled, x, y) + return v0.Assign(blobData, dictStore, eip4844Enabled, x, y) } err = errors.New("decompression circuit assignment : unsupported blob version") return diff --git a/prover/circuits/blobdecompression/v0/assign_test.go b/prover/circuits/blobdecompression/v0/assign_test.go index 3c99b858f..71a5c7442 100644 --- a/prover/circuits/blobdecompression/v0/assign_test.go +++ b/prover/circuits/blobdecompression/v0/assign_test.go @@ -23,7 +23,11 @@ import ( ) func TestBlobV0(t *testing.T) { - resp, blobBytes, dict := mustGetTestCompressedData(t) + dict := lzss.AugmentDict(test_utils.GetDict(t)) + dictStore, err := dictionary.SingletonStore(dict, 0) + assert.NoError(t, err) + + resp, blobBytes := mustGetTestCompressedData(t, dictStore) circ := v0.Allocate(dict) logrus.Infof("Building the constraint system") @@ -46,7 +50,7 @@ func TestBlobV0(t *testing.T) { givenSnarkHash, err := utils.HexDecodeString(resp.SnarkHash) assert.NoError(t, err) - a, _, snarkHash, err := blobdecompression.Assign(blobBytes, dict, true, x, y) + a, _, snarkHash, err := blobdecompression.Assign(blobBytes, dictStore, true, x, y) assert.NoError(t, err) _, ok := a.(*v0.Circuit) assert.True(t, ok) @@ -64,9 +68,7 @@ func TestBlobV0(t *testing.T) { // mustGetTestCompressedData is a test utility function that we use to get // actual compressed data from the -func mustGetTestCompressedData(t *testing.T) (resp blobsubmission.Response, blobBytes []byte, dict []byte) { - dict = lzss.AugmentDict(test_utils.GetDict(t)) - +func mustGetTestCompressedData(t *testing.T, dictStore dictionary.Store) (resp blobsubmission.Response, blobBytes []byte) { respJson, err := os.ReadFile("sample-blob.json") assert.NoError(t, err) @@ -75,8 +77,6 @@ func mustGetTestCompressedData(t *testing.T) (resp blobsubmission.Response, blob blobBytes, err = base64.StdEncoding.DecodeString(resp.CompressedData) assert.NoError(t, err) - dictStore, err := dictionary.SingletonStore(dict, 0) - assert.NoError(t, err) _, _, _, err = blob.DecompressBlob(blobBytes, dictStore) assert.NoError(t, err) diff --git a/prover/circuits/blobdecompression/v0/prelude.go b/prover/circuits/blobdecompression/v0/prelude.go index fa00d73e7..0a66c4c7e 100644 --- a/prover/circuits/blobdecompression/v0/prelude.go +++ b/prover/circuits/blobdecompression/v0/prelude.go @@ -46,7 +46,7 @@ func MakeCS(dict []byte) constraint.ConstraintSystem { // Assign the circuit with concrete data. Returns the assigned circuit and the // public input computed during the assignment. // @alexandre.belling should we instead compute snarkHash independently here? Seems like it doesn't need to be included in the req received by Prove -func Assign(blobData, dict []byte, eip4844Enabled bool, x [32]byte, y fr381.Element) (assignment frontend.Circuit, publicInput fr.Element, snarkHash []byte, err error) { +func Assign(blobData []byte, dictStore dictionary.Store, eip4844Enabled bool, x [32]byte, y fr381.Element) (assignment frontend.Circuit, publicInput fr.Element, snarkHash []byte, err error) { const maxCLen = blob.MaxUsableBytes const maxDLen = blob.MaxUncompressedBytes @@ -56,11 +56,6 @@ func Assign(blobData, dict []byte, eip4844Enabled bool, x [32]byte, y fr381.Elem return } - dictStore, err := dictionary.SingletonStore(dict, 0) - if err != nil { - err = fmt.Errorf("failed to create dictionary store %w", err) - return - } header, uncompressedData, _, err := blob.DecompressBlob(blobData, dictStore) if err != nil { err = fmt.Errorf("decompression circuit assignment : could not decompress the data : %w", err) diff --git a/prover/circuits/blobdecompression/v1/assign_test.go b/prover/circuits/blobdecompression/v1/assign_test.go index 8bf6c2ea1..e42b26747 100644 --- a/prover/circuits/blobdecompression/v1/assign_test.go +++ b/prover/circuits/blobdecompression/v1/assign_test.go @@ -30,7 +30,7 @@ func prepare(t require.TestingT, blobBytes []byte) (c *v1.Circuit, a frontend.Ci dictStore, err := dictionary.SingletonStore(blobtestutils.GetDict(t), 1) assert.NoError(t, err) - _, payload, _, err := blobcompressorv1.DecompressBlob(blobBytes, dictStore) + _, payload, _, dict, err := blobcompressorv1.DecompressBlob(blobBytes, dictStore) assert.NoError(t, err) resp, err := blobsubmission.CraftResponse(&blobsubmission.Request{ @@ -50,8 +50,7 @@ func prepare(t require.TestingT, blobBytes []byte) (c *v1.Circuit, a frontend.Ci y.SetBytes(b) blobBytes = append(blobBytes, make([]byte, blobcompressorv1.MaxUsableBytes-len(blobBytes))...) - dict := blobtestutils.GetDict(t) - a, _, snarkHash, err := blobdecompression.Assign(blobBytes, dict, true, x, y) + a, _, snarkHash, err := blobdecompression.Assign(blobBytes, dictStore, true, x, y) assert.NoError(t, err) _, ok := a.(*v1.Circuit) diff --git a/prover/circuits/blobdecompression/v1/circuit.go b/prover/circuits/blobdecompression/v1/circuit.go index a04669002..405e6a07c 100644 --- a/prover/circuits/blobdecompression/v1/circuit.go +++ b/prover/circuits/blobdecompression/v1/circuit.go @@ -247,18 +247,13 @@ func Compile(dictionaryLength int) constraint.ConstraintSystem { } } -func AssignFPI(blobBytes, dict []byte, eip4844Enabled bool, x [32]byte, y fr381.Element) (fpi FunctionalPublicInput, err error) { +func AssignFPI(blobBytes []byte, dictStore dictionary.Store, eip4844Enabled bool, x [32]byte, y fr381.Element) (fpi FunctionalPublicInput, dict []byte, err error) { if len(blobBytes) != blob.MaxUsableBytes { err = fmt.Errorf("decompression circuit assignment : invalid blob length : %d. expected %d", len(blobBytes), blob.MaxUsableBytes) return } - dictStore, err := dictionary.SingletonStore(dict, 1) - if err != nil { - err = fmt.Errorf("failed to create dictionary store %w", err) - return - } - header, payload, _, err := blob.DecompressBlob(blobBytes, dictStore) + header, payload, _, dict, err := blob.DecompressBlob(blobBytes, dictStore) if err != nil { return } @@ -294,9 +289,9 @@ func AssignFPI(blobBytes, dict []byte, eip4844Enabled bool, x [32]byte, y fr381. return } -func Assign(blobBytes, dict []byte, eip4844Enabled bool, x [32]byte, y fr381.Element) (assignment frontend.Circuit, publicInput fr377.Element, snarkHash []byte, err error) { +func Assign(blobBytes []byte, dictStore dictionary.Store, eip4844Enabled bool, x [32]byte, y fr381.Element) (assignment frontend.Circuit, publicInput fr377.Element, snarkHash []byte, err error) { - fpi, err := AssignFPI(blobBytes, dict, eip4844Enabled, x, y) + fpi, dict, err := AssignFPI(blobBytes, dictStore, eip4844Enabled, x, y) if err != nil { return } diff --git a/prover/circuits/blobdecompression/v1/snark_test.go b/prover/circuits/blobdecompression/v1/snark_test.go index d70724cc1..778e923f2 100644 --- a/prover/circuits/blobdecompression/v1/snark_test.go +++ b/prover/circuits/blobdecompression/v1/snark_test.go @@ -55,7 +55,7 @@ func TestParseHeader(t *testing.T) { for _, blobData := range blobs { - header, _, blocks, err := blob.DecompressBlob(blobData, dictStore) + header, _, blocks, _, err := blob.DecompressBlob(blobData, dictStore) assert.NoError(t, err) assert.LessOrEqual(t, len(blocks), MaxNbBatches, "too many batches") @@ -347,7 +347,7 @@ func TestDictHash(t *testing.T) { dict := blobtestutils.GetDict(t) dictStore, err := dictionary.SingletonStore(blobtestutils.GetDict(t), 1) assert.NoError(t, err) - header, _, _, err := blob.DecompressBlob(blobBytes, dictStore) // a bit roundabout, but the header field is not public + header, _, _, _, err := blob.DecompressBlob(blobBytes, dictStore) // a bit roundabout, but the header field is not public assert.NoError(t, err) circuit := testDataDictHashCircuit{ diff --git a/prover/circuits/pi-interconnection/assign.go b/prover/circuits/pi-interconnection/assign.go index a817c4736..9e231e262 100644 --- a/prover/circuits/pi-interconnection/assign.go +++ b/prover/circuits/pi-interconnection/assign.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/base64" "fmt" + "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/dictionary" "hash" "github.com/consensys/linea-monorepo/prover/crypto/mimc" @@ -13,7 +14,6 @@ import ( decompression "github.com/consensys/linea-monorepo/prover/circuits/blobdecompression/v1" "github.com/consensys/linea-monorepo/prover/circuits/internal" "github.com/consensys/linea-monorepo/prover/circuits/pi-interconnection/keccak" - "github.com/consensys/linea-monorepo/prover/lib/compressor/blob" public_input "github.com/consensys/linea-monorepo/prover/public-input" "github.com/consensys/linea-monorepo/prover/utils" "github.com/sirupsen/logrus" @@ -24,12 +24,9 @@ type Request struct { Decompressions []blobsubmission.Response Executions []public_input.Execution Aggregation public_input.Aggregation - // Path to the compression dictionary. Used to extract the execution data - // for each execution. - DictPath string } -func (c *Compiled) Assign(r Request) (a Circuit, err error) { +func (c *Compiled) Assign(r Request, dictStore dictionary.Store) (a Circuit, err error) { internal.RegisterHints() keccak.RegisterHints() utils.RegisterHints() @@ -56,13 +53,6 @@ func (c *Compiled) Assign(r Request) (a Circuit, err error) { return } - // @alex: We should pass that as a parameter. And also (@arya) pass a list - // of dictionnary because this function. - dict, err := blob.GetDict(r.DictPath) - if err != nil { - return Circuit{}, fmt.Errorf("could not find the dictionnary: path=%v err=%v", r.DictPath, err) - } - // For Shnarfs and Merkle Roots hshK := c.Keccak.GetHasher() @@ -111,7 +101,7 @@ func (c *Compiled) Assign(r Request) (a Circuit, err error) { fpi decompression.FunctionalPublicInput sfpi decompression.FunctionalPublicInputSnark ) - if fpi, err = decompression.AssignFPI(blobData[:], dict, p.Eip4844Enabled, x, y); err != nil { + if fpi, _, err = decompression.AssignFPI(blobData[:], dictStore, p.Eip4844Enabled, x, y); err != nil { return } execDataChecksums = append(execDataChecksums, fpi.BatchSums...) // len(execDataChecksums) = index of the first execution associated with the next blob diff --git a/prover/circuits/pi-interconnection/bench/main.go b/prover/circuits/pi-interconnection/bench/main.go deleted file mode 100644 index e136255d4..000000000 --- a/prover/circuits/pi-interconnection/bench/main.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/consensys/linea-monorepo/prover/utils/test_utils" - - "github.com/consensys/gnark-crypto/ecc" - "github.com/consensys/gnark/backend/plonk" - "github.com/consensys/gnark/frontend" - "github.com/consensys/gnark/frontend/cs/scs" - "github.com/consensys/gnark/test/unsafekzg" - pi_interconnection "github.com/consensys/linea-monorepo/prover/circuits/pi-interconnection" - pitesting "github.com/consensys/linea-monorepo/prover/circuits/pi-interconnection/test_utils" - "github.com/consensys/linea-monorepo/prover/config" - - "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" - "github.com/stretchr/testify/assert" -) - -func main() { - var b test_utils.FakeTestingT - req := pitesting.AssignSingleBlockBlob(b) - - c, err := pi_interconnection.Compile(config.PublicInput{ - MaxNbDecompression: 400, - MaxNbExecution: 400, - ExecutionMaxNbMsg: 16, - L2MsgMerkleDepth: 5, - L2MsgMaxNbMerkle: 10, - }, dummy.Compile) // note that the solving/proving time will not reflect the wizard proof or verification - assert.NoError(b, err) - - a, err := c.Assign(req) - assert.NoError(b, err) - - c.Circuit.UseGkrMimc = true - - cs, err := frontend.Compile(ecc.BLS12_377.ScalarField(), scs.NewBuilder, c.Circuit, frontend.WithCapacity(40_000_000)) - assert.NoError(b, err) - - kzgc, kzgl, err := unsafekzg.NewSRS(cs) - assert.NoError(b, err) - - pk, _, err := plonk.Setup(cs, kzgc, kzgl) - assert.NoError(b, err) - - secondsStart := time.Now().Unix() - - w, err := frontend.NewWitness(&a, ecc.BLS12_377.ScalarField()) - assert.NoError(b, err) - _, err = plonk.Prove(cs, pk, w) - assert.NoError(b, err) - - fmt.Println(time.Now().Unix()-secondsStart, "seconds") -} diff --git a/prover/circuits/pi-interconnection/e2e_test.go b/prover/circuits/pi-interconnection/e2e_test.go index ca53f69ae..50e9c4673 100644 --- a/prover/circuits/pi-interconnection/e2e_test.go +++ b/prover/circuits/pi-interconnection/e2e_test.go @@ -5,6 +5,7 @@ package pi_interconnection_test import ( "encoding/base64" "fmt" + "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/dictionary" "slices" "testing" @@ -47,7 +48,10 @@ func TestSingleBlockBlobE2E(t *testing.T) { compiled, err := pi_interconnection.Compile(cfg, dummy.Compile) assert.NoError(t, err) - a, err := compiled.Assign(req) + dictStore, err := dictionary.SingletonStore(blobtesting.GetDict(t), 1) + assert.NoError(t, err) + + a, err := compiled.Assign(req, dictStore) assert.NoError(t, err) cs, err := frontend.Compile(ecc.BLS12_377.ScalarField(), scs.NewBuilder, compiled.Circuit, frontend.WithCapacity(3_000_000)) @@ -112,7 +116,6 @@ func TestTinyTwoBatchBlob(t *testing.T) { req := pi_interconnection.Request{ Decompressions: []blobsubmission.Response{*blobResp}, Executions: execReq, - DictPath: "../../lib/compressor/compressor_dict.bin", Aggregation: public_input.Aggregation{ FinalShnarf: blobResp.ExpectedShnarf, ParentAggregationFinalShnarf: blobReq.PrevShnarf, @@ -209,7 +212,6 @@ func TestTwoTwoBatchBlobs(t *testing.T) { req := pi_interconnection.Request{ Decompressions: []blobsubmission.Response{*blobResp0, *blobResp1}, Executions: execReq, - DictPath: "../../lib/compressor/compressor_dict.bin", Aggregation: public_input.Aggregation{ FinalShnarf: blobResp1.ExpectedShnarf, ParentAggregationFinalShnarf: blobReq0.PrevShnarf, @@ -255,6 +257,9 @@ func testPI(t *testing.T, req pi_interconnection.Request, options ...testPIOptio slackIterationNum := len(cfg.slack) * len(cfg.slack) slackIterationNum *= slackIterationNum + dictStore, err := dictionary.SingletonStore(blobtesting.GetDict(t), 1) + assert.NoError(t, err) + var slack [4]int for i := 0; i < slackIterationNum; i++ { @@ -277,7 +282,7 @@ func testPI(t *testing.T, req pi_interconnection.Request, options ...testPIOptio compiled, err := pi_interconnection.Compile(cfg, dummy.Compile) assert.NoError(t, err) - a, err := compiled.Assign(req) + a, err := compiled.Assign(req, dictStore) assert.NoError(t, err) assert.NoError(t, test.IsSolved(compiled.Circuit, &a, ecc.BLS12_377.ScalarField())) diff --git a/prover/circuits/pi-interconnection/test_utils/test_utils.go b/prover/circuits/pi-interconnection/test_utils/test_utils.go index 36f1162ed..c678f1673 100644 --- a/prover/circuits/pi-interconnection/test_utils/test_utils.go +++ b/prover/circuits/pi-interconnection/test_utils/test_utils.go @@ -48,7 +48,6 @@ func AssignSingleBlockBlob(t require.TestingT) pi_interconnection.Request { merkleRoots := aggregation.PackInMiniTrees(test_utils.BlocksToHex(execReq.L2MessageHashes)) return pi_interconnection.Request{ - DictPath: "../../lib/compressor/compressor_dict.bin", Decompressions: []blobsubmission.Response{*blobResp}, Executions: []public_input.Execution{execReq}, Aggregation: public_input.Aggregation{ diff --git a/prover/cmd/prover/cmd/setup.go b/prover/cmd/prover/cmd/setup.go index b167292db..61676e88a 100644 --- a/prover/cmd/prover/cmd/setup.go +++ b/prover/cmd/prover/cmd/setup.go @@ -3,6 +3,7 @@ package cmd import ( "context" "crypto/sha256" + "errors" "fmt" "io" "os" @@ -32,7 +33,8 @@ import ( type SetupArgs struct { Force bool Circuits string - DictPath string + DictPath string // to be deprecated; only used for compiling v0 blob decompression circuit + DictSize int AssetsDir string ConfigFile string } @@ -66,7 +68,7 @@ func Setup(context context.Context, args SetupArgs) error { // parse inCircuits inCircuits := make(map[circuits.CircuitID]bool) for _, c := range AllCircuits { - inCircuits[circuits.CircuitID(c)] = false + inCircuits[c] = false } _inCircuits := strings.Split(args.Circuits, ",") for _, c := range _inCircuits { @@ -85,6 +87,8 @@ func Setup(context context.Context, args SetupArgs) error { if err != nil { return fmt.Errorf("%s failed to create SRS provider: %w", cmdName, err) } + var foundDecompressionV0 bool // this is a temporary mechanism to make sure we phase out the practice + // of providing entire dictionaries for setup. // for each circuit, we start by compiling the circuit // then we do a sha sum and compare against the one in the manifest.json @@ -98,7 +102,6 @@ func Setup(context context.Context, args SetupArgs) error { logrus.Infof("setting up %s", c) var builder circuits.Builder - var dict []byte extraFlags := make(map[string]any) // let's compile the circuit. @@ -111,21 +114,21 @@ func Setup(context context.Context, args SetupArgs) error { extraFlags["cfg_checksum"] = limits.Checksum() zkEvm := zkevm.FullZkEvm(&limits) builder = execution.NewBuilder(zkEvm) - case circuits.BlobDecompressionV0CircuitID, circuits.BlobDecompressionV1CircuitID: - dict, err = os.ReadFile(args.DictPath) + + case circuits.BlobDecompressionV0CircuitID: + dict, err := os.ReadFile(args.DictPath) if err != nil { return fmt.Errorf("%s failed to read dictionary file: %w", cmdName, err) } + foundDecompressionV0 = true + extraFlags["maxUsableBytes"] = blob_v0.MaxUsableBytes + extraFlags["maxUncompressedBytes"] = blob_v0.MaxUncompressedBytes + builder = v0.NewBuilder(dict) + case circuits.BlobDecompressionV1CircuitID: + extraFlags["maxUsableBytes"] = blob_v1.MaxUsableBytes + extraFlags["maxUncompressedBytes"] = blob_v1.MaxUncompressedBytes + builder = v1.NewBuilder(args.DictSize) - if c == circuits.BlobDecompressionV0CircuitID { - extraFlags["maxUsableBytes"] = blob_v0.MaxUsableBytes - extraFlags["maxUncompressedBytes"] = blob_v0.MaxUncompressedBytes - builder = v0.NewBuilder(dict) - } else if c == circuits.BlobDecompressionV1CircuitID { - extraFlags["maxUsableBytes"] = blob_v1.MaxUsableBytes - extraFlags["maxUncompressedBytes"] = blob_v1.MaxUncompressedBytes - builder = v1.NewBuilder(len(dict)) - } case circuits.PublicInputInterconnectionCircuitID: builder = pi_interconnection.NewBuilder(cfg.PublicInputInterconnection) case circuits.EmulationDummyCircuitID: @@ -138,14 +141,10 @@ func Setup(context context.Context, args SetupArgs) error { if err := updateSetup(context, cfg, args.Force, srsProvider, c, builder, extraFlags); err != nil { return err } - if dict != nil { - // we save the dictionary to disk - dictPath := cfg.BlobDecompressionDictPath(string(c)) - if err := os.WriteFile(dictPath, dict, 0600); err != nil { - return fmt.Errorf("%s failed to write dictionary file: %w", cmdName, err) - } - } + } + if !foundDecompressionV0 && args.DictPath != "" { + return errors.New("explicit provision of a dictionary is only allowed for backwards compatibility with v0 blob decompression") } if !(inCircuits[circuits.AggregationCircuitID] || inCircuits[circuits.EmulationCircuitID]) { diff --git a/prover/cmd/prover/main.go b/prover/cmd/prover/main.go index 073110b08..fcd519b3f 100644 --- a/prover/cmd/prover/main.go +++ b/prover/cmd/prover/main.go @@ -55,7 +55,8 @@ func init() { rootCmd.AddCommand(setupCmd) setupCmd.Flags().BoolVar(&setupArgs.Force, "force", false, "overwrites existing files") setupCmd.Flags().StringVar(&setupArgs.Circuits, "circuits", strings.Join(allCircuitList(), ","), "comma separated list of circuits to setup") - setupCmd.Flags().StringVar(&setupArgs.DictPath, "dict", "", "path to the dictionary file used in blob (de)compression") + setupCmd.Flags().StringVar(&setupArgs.DictPath, "dict", "", "path to the dictionary file used in blob (de)compression (for v0 only)") + setupCmd.Flags().IntVar(&setupArgs.DictSize, "dict-size", 65536, "size in bytes of the dictionary used in blob (de)compression") setupCmd.Flags().StringVar(&setupArgs.AssetsDir, "assets-dir", "", "path to the directory where the assets are stored (override conf)") viper.BindPFlag("assets_dir", setupCmd.Flags().Lookup("assets-dir")) diff --git a/prover/config/config-sepolia-full.toml b/prover/config/config-sepolia-full.toml index b27aff0f1..452cbae71 100644 --- a/prover/config/config-sepolia-full.toml +++ b/prover/config/config-sepolia-full.toml @@ -1,7 +1,7 @@ environment = "sepolia" version = "4.0.0" # TODO @gbotrel hunt all version definitions. assets_dir = "./prover-assets" -log_level = 4 # TODO @gbotrel will be refactored with new logger. +log_level = 4 # TODO @gbotrel will be refactored with new logger. [controller] retry_delays = [0, 1] @@ -15,7 +15,7 @@ requests_root_dir = "/home/ubuntu/sepolia-testing-full/prover-execution" [blob_decompression] prover_mode = "full" requests_root_dir = "/home/ubuntu/sepolia-testing-full/prover-compression" -dict_path = "lib/compressor/compressor_dict.bin" +dict_paths = ["lib/compressor/compressor_dict.bin"] [aggregation] prover_mode = "full" @@ -134,4 +134,4 @@ BLOCK_L2_L1_LOGS = 16 BLOCK_TRANSACTIONS = 200 BIN_REFERENCE_TABLE = 262144 SHF_REFERENCE_TABLE = 4096 -INSTRUCTION_DECODER = 512 \ No newline at end of file +INSTRUCTION_DECODER = 512 diff --git a/prover/config/config.go b/prover/config/config.go index 2a0ee91e3..13eb40302 100644 --- a/prover/config/config.go +++ b/prover/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/dictionary" "os" "path" "path/filepath" @@ -218,13 +219,13 @@ type BlobDecompression struct { // ProverMode stores the kind of prover to use. ProverMode ProverMode `mapstructure:"prover_mode" validate:"required,oneof=dev full"` - // DictPath is an optional parameters allowing the user to specificy explicitly - // where to look for the compression dictionary. If the input is not provided + // DictPaths is an optional parameters allowing the user to specify explicitly + // where to look for the compression dictionaries. If the input is not provided // then the dictionary will be fetched in ///compression_dict.bin. // // We stress that the feature should not be used in production and should - // only be used in E2E testing context. - DictPath string `mapstructure:"dict_path"` + // only be used in E2E testing context. TODO @Tabaie @alexandre.belling revise this warning, seems to no longer apply + DictPaths []string `mapstructure:"dict_paths"` } type Aggregation struct { @@ -280,15 +281,16 @@ type PublicInput struct { } -// BlobDecompressionDictPath returns the filepath where to look for the blob -// decompression dictionary file. If provided in the config, the function returns -// in priority the provided [BlobDecompression.DictPath] or it returns a +// BlobDecompressionDictStore returns a decompression dictionary store +// loaded from paths specified in [BlobDecompression.DictPaths]. +// If no such path is provided, it loads one from the // prover assets path depending on the provided circuitID. -func (cfg *Config) BlobDecompressionDictPath(circuitID string) string { +func (cfg *Config) BlobDecompressionDictStore(circuitID string) dictionary.Store { - if len(cfg.BlobDecompression.DictPath) > 0 { - return cfg.BlobDecompression.DictPath + paths := cfg.BlobDecompression.DictPaths + if len(paths) == 0 { + paths = []string{filepath.Join(cfg.PathForSetup(circuitID), DefaultDictionaryFileName)} } - return filepath.Join(cfg.PathForSetup(string(circuitID)), DefaultDictionaryFileName) + return dictionary.NewStore(paths...) } diff --git a/prover/lib/compressor/blob/blob.go b/prover/lib/compressor/blob/blob.go index 89e7dbdaf..0d1e5d0e3 100644 --- a/prover/lib/compressor/blob/blob.go +++ b/prover/lib/compressor/blob/blob.go @@ -23,7 +23,7 @@ func GetVersion(blob []byte) uint16 { return 0 } -func GetDict(dictPath string) ([]byte, error) { +func LoadDict(dictPath string) ([]byte, error) { return os.ReadFile(dictPath) } @@ -43,7 +43,7 @@ func DecompressBlob(blob []byte, dictStore dictionary.Store) ([]byte, error) { _, _, blocks, err = v0.DecompressBlob(blob, dictStore) blockDecoder = v0.DecodeBlockFromUncompressed case 1: - _, _, blocks, err = v1.DecompressBlob(blob, dictStore) + _, _, blocks, _, err = v1.DecompressBlob(blob, dictStore) blockDecoder = v1.DecodeBlockFromUncompressed default: return nil, errors.New("unrecognized blob version") diff --git a/prover/lib/compressor/blob/v1/blob_maker.go b/prover/lib/compressor/blob/v1/blob_maker.go index fd93d6493..de8fa80ac 100644 --- a/prover/lib/compressor/blob/v1/blob_maker.go +++ b/prover/lib/compressor/blob/v1/blob_maker.go @@ -115,7 +115,7 @@ func (bm *BlobMaker) Written() int { func (bm *BlobMaker) Bytes() []byte { if bm.currentBlobLength > 0 { // sanity check that we can always decompress. - header, rawBlocks, _, err := DecompressBlob(bm.currentBlob[:bm.currentBlobLength], bm.dictStore) + header, rawBlocks, _, _, err := DecompressBlob(bm.currentBlob[:bm.currentBlobLength], bm.dictStore) if err != nil { var sbb strings.Builder fmt.Fprintf(&sbb, "invalid blob: %v\n", err) @@ -302,23 +302,22 @@ func (bm *BlobMaker) Equals(other *BlobMaker) bool { } // DecompressBlob decompresses a blob and returns the header and the blocks as they were compressed. -func DecompressBlob(b []byte, dictStore dictionary.Store) (blobHeader *Header, rawPayload []byte, blocks [][]byte, err error) { +func DecompressBlob(b []byte, dictStore dictionary.Store) (blobHeader *Header, rawPayload []byte, blocks [][]byte, dict []byte, err error) { // UnpackAlign the blob b, err = encode.UnpackAlign(b, fr381.Bits-1, false) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } // read the header blobHeader = new(Header) read, err := blobHeader.ReadFrom(bytes.NewReader(b)) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to read blob header: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to read blob header: %w", err) } // retrieve dict - dict, err := dictStore.Get(blobHeader.DictChecksum[:], 1) - if err != nil { - return nil, nil, nil, err + if dict, err = dictStore.Get(blobHeader.DictChecksum[:], 1); err != nil { + return nil, nil, nil, nil, err } b = b[read:] @@ -326,7 +325,7 @@ func DecompressBlob(b []byte, dictStore dictionary.Store) (blobHeader *Header, r // decompress the data rawPayload, err = lzss.Decompress(b, dict) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to decompress blob body: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to decompress blob body: %w", err) } offset := 0 @@ -335,7 +334,7 @@ func DecompressBlob(b []byte, dictStore dictionary.Store) (blobHeader *Header, r batchOffset := offset for offset < batchOffset+batchLen { if blockLen, err := ScanBlockByteLen(rawPayload[offset:]); err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } else { blocks = append(blocks, rawPayload[offset:offset+blockLen]) offset += blockLen @@ -343,11 +342,11 @@ func DecompressBlob(b []byte, dictStore dictionary.Store) (blobHeader *Header, r } if offset != batchOffset+batchLen { - return nil, nil, nil, errors.New("incorrect batch length") + return nil, nil, nil, nil, errors.New("incorrect batch length") } } - return blobHeader, rawPayload, blocks, nil + return blobHeader, rawPayload, blocks, dict, nil } // WorstCompressedBlockSize returns the size of the given block, as compressed by an "empty" blob maker. diff --git a/prover/lib/compressor/blob/v1/blob_maker_test.go b/prover/lib/compressor/blob/v1/blob_maker_test.go index cdf387c6e..3c8695d7a 100644 --- a/prover/lib/compressor/blob/v1/blob_maker_test.go +++ b/prover/lib/compressor/blob/v1/blob_maker_test.go @@ -7,6 +7,7 @@ import ( cRand "crypto/rand" "encoding/binary" "encoding/hex" + "github.com/consensys/linea-monorepo/prover/utils/test_utils" "math/big" "math/rand/v2" "os" @@ -17,7 +18,7 @@ import ( "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode" v1 "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1" - "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1/test_utils" + v1Testing "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1/test_utils" fr381 "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" @@ -55,7 +56,7 @@ func testCompressorSingleSmallBatch(t *testing.T, blocks [][]byte) { dict, err := os.ReadFile(testDictPath) assert.NoError(t, err) dictStore, err := dictionary.SingletonStore(dict, 1) - _, _, blocksBack, err := v1.DecompressBlob(bm.Bytes(), dictStore) + _, _, blocksBack, _, err := v1.DecompressBlob(bm.Bytes(), dictStore) assert.NoError(t, err) assert.Equal(t, len(blocks), len(blocksBack), "number of blocks should match") // TODO compare the blocks @@ -484,7 +485,7 @@ func init() { panic(err) } - if testBlocks, err = test_utils.LoadTestBlocks(filepath.Join(rootPath, "testdata/prover-v2/prover-execution/requests")); err != nil { + if testBlocks, err = v1Testing.LoadTestBlocks(filepath.Join(rootPath, "testdata/prover-v2/prover-execution/requests")); err != nil { panic(err) } diff --git a/prover/lib/compressor/blob/v1/encode_test.go b/prover/lib/compressor/blob/v1/encode_test.go index 9e36a05bd..fdb5d9b68 100644 --- a/prover/lib/compressor/blob/v1/encode_test.go +++ b/prover/lib/compressor/blob/v1/encode_test.go @@ -247,7 +247,7 @@ func decompressBlob(b []byte) ([][][]byte, error) { if err != nil { return nil, err } - header, _, blocks, err := v1.DecompressBlob(b, dictStore) + header, _, blocks, _, err := v1.DecompressBlob(b, dictStore) if err != nil { return nil, fmt.Errorf("can't decompress blob: %w", err) } diff --git a/prover/lib/compressor/blob/v1/test_utils/blob_maker_testing.go b/prover/lib/compressor/blob/v1/test_utils/blob_maker_testing.go index ca04fbf8e..7507fdb49 100644 --- a/prover/lib/compressor/blob/v1/test_utils/blob_maker_testing.go +++ b/prover/lib/compressor/blob/v1/test_utils/blob_maker_testing.go @@ -5,7 +5,7 @@ import ( "crypto/rand" "encoding/binary" "encoding/json" - "errors" + "github.com/consensys/linea-monorepo/prover/utils/test_utils" "os" "path/filepath" "strings" @@ -125,7 +125,7 @@ func ConsecutiveBlobs(t require.TestingT, n ...int) [][]byte { } func TestBlocksAndBlobMaker(t require.TestingT) ([][]byte, *v1.BlobMaker) { - repoRoot, err := GetRepoRootPath() + repoRoot, err := test_utils.GetRepoRootPath() assert.NoError(t, err) testBlocks, err := LoadTestBlocks(filepath.Join(repoRoot, "testdata/prover-v2/prover-execution/requests")) assert.NoError(t, err) @@ -141,23 +141,8 @@ func GetDict(t require.TestingT) []byte { return dict } -// GetRepoRootPath assumes that current working directory is within the repo -func GetRepoRootPath() (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - const repoName = "linea-monorepo" - i := strings.LastIndex(wd, repoName) - if i == -1 { - return "", errors.New("could not find repo root") - } - i += len(repoName) - return wd[:i], nil -} - func getDictForTest() ([]byte, error) { - repoRoot, err := GetRepoRootPath() + repoRoot, err := test_utils.GetRepoRootPath() if err != nil { return nil, err } diff --git a/prover/lib/compressor/dict/25-01-15-INCOMPLETE.bin b/prover/lib/compressor/dict/25-01-15-INCOMPLETE.bin new file mode 100644 index 000000000..2422d663c Binary files /dev/null and b/prover/lib/compressor/dict/25-01-15-INCOMPLETE.bin differ diff --git a/prover/utils/test_utils/test_utils.go b/prover/utils/test_utils/test_utils.go index 74616356b..473152f2a 100644 --- a/prover/utils/test_utils/test_utils.go +++ b/prover/utils/test_utils/test_utils.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "errors" "fmt" "hash" "io" @@ -347,3 +348,18 @@ func spaceOutFromRight(s string) string { } return bb.String() } + +// GetRepoRootPath assumes that current working directory is within the repo +func GetRepoRootPath() (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + const repoName = "linea-monorepo" + i := strings.LastIndex(wd, repoName) + if i == -1 { + return "", errors.New("could not find repo root") + } + i += len(repoName) + return wd[:i], nil +}