From 49d0f3830c37f980f3a8ac07271f497283164b6d Mon Sep 17 00:00:00 2001 From: Steven Landers Date: Mon, 17 Jun 2024 16:46:17 -0400 Subject: [PATCH 1/6] add compression for receipts --- Makefile | 4 + go.mod | 8 +- go.sum | 11 ++ proto/evm/compression.proto | 18 +++ utils/compression/compress.go | 51 ++++++ utils/compression/compress_test.go | 242 +++++++++++++++++++++++++++++ utils/compression/zlib.go | 34 ++++ x/evm/keeper/receipt.go | 9 +- 8 files changed, 371 insertions(+), 6 deletions(-) create mode 100644 proto/evm/compression.proto create mode 100644 utils/compression/compress.go create mode 100644 utils/compression/compress_test.go create mode 100644 utils/compression/zlib.go diff --git a/Makefile b/Makefile index 9fe570606..7706706ca 100644 --- a/Makefile +++ b/Makefile @@ -220,3 +220,7 @@ split-test-packages:$(BUILDDIR)/packages.txt split -d -n l/$(NUM_SPLIT) $< $<. test-group-%:split-test-packages cat $(BUILDDIR)/packages.txt.$* | xargs go test -parallel 4 -mod=readonly -timeout=10m -race -coverprofile=$*.profile.out -covermode=atomic + +.PHONY: proto +proto: + ignite generate proto-go -y \ No newline at end of file diff --git a/go.mod b/go.mod index ce067a213..c6e224543 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,20 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/CosmWasm/wasmd v0.27.0 github.com/CosmWasm/wasmvm v1.5.2 + github.com/andybalholm/brotli v1.1.0 github.com/armon/go-metrics v0.4.1 github.com/btcsuite/btcd v0.22.1 github.com/cosmos/cosmos-sdk v0.45.10 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583 github.com/cosmos/ibc-go/v3 v3.0.0 + github.com/dsnet/compress v0.0.1 github.com/ethereum/go-ethereum v1.13.2 github.com/go-playground/validator/v10 v10.4.1 github.com/gogo/protobuf v1.3.3 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/golang/protobuf v1.5.4 + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/golangci/golangci-lint v1.46.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 @@ -24,7 +27,9 @@ require ( github.com/holiman/uint256 v1.2.4 github.com/justinas/alice v1.2.0 github.com/k0kubun/pp/v3 v3.2.0 + github.com/klauspost/compress v1.16.3 github.com/mitchellh/mapstructure v1.5.0 + github.com/pierrec/lz4/v4 v4.1.21 github.com/pkg/errors v0.9.1 github.com/rakyll/statik v0.1.7 github.com/rs/cors v1.8.2 @@ -38,6 +43,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d github.com/tendermint/tendermint v0.37.0-dev github.com/tendermint/tm-db v0.6.8-0.20220519162814-e24b96538a12 + github.com/ulikunitz/xz v0.5.6 go.opentelemetry.io/otel v1.9.0 go.opentelemetry.io/otel/trace v1.9.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa @@ -149,7 +155,6 @@ require ( github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect @@ -198,7 +203,6 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kisielk/errcheck v1.6.0 // indirect github.com/kisielk/gotool v1.0.0 // indirect - github.com/klauspost/compress v1.16.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kulti/thelper v0.6.2 // indirect diff --git a/go.sum b/go.sum index 48d365910..e8c82ace7 100644 --- a/go.sum +++ b/go.sum @@ -155,6 +155,8 @@ github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= @@ -386,6 +388,9 @@ github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRP github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -921,6 +926,7 @@ github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -929,6 +935,7 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1207,6 +1214,8 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= @@ -1514,6 +1523,8 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.5 h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqzi/CzI= diff --git a/proto/evm/compression.proto b/proto/evm/compression.proto new file mode 100644 index 000000000..ced18b90a --- /dev/null +++ b/proto/evm/compression.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +package seiprotocol.seichain.evm; + +import "gogoproto/gogo.proto"; + +option go_package = "github.com/sei-protocol/sei-chain/x/evm/types"; + +// CompressedData is only used for receipts as the moment +// It is not meant to be used outside of storage and retrieval +message CompressedData { + enum Algorithm { + ZLIB = 0; + } + + // if true, it informs the client that the data is compressed + bytes data = 1; + Algorithm algorithm = 2; +} \ No newline at end of file diff --git a/utils/compression/compress.go b/utils/compression/compress.go new file mode 100644 index 000000000..0021684f0 --- /dev/null +++ b/utils/compression/compress.go @@ -0,0 +1,51 @@ +package compression + +import ( + "fmt" + "github.com/gogo/protobuf/proto" + "github.com/sei-protocol/sei-chain/x/evm/types" +) + +func CompressMessage(message proto.Message) ([]byte, error) { + b, err := proto.Marshal(message) + if err != nil { + return nil, err + } + + bCompressed, err := compressZLib(b) + if err != nil { + return nil, err + } + + cd := &types.CompressedData{ + Data: bCompressed, + Algorithm: types.CompressedData_ZLIB, + } + return cd.Marshal() +} + +func DecompressMessage(target proto.Message, compressed []byte) error { + // this will work if the type is CompressedData + // if not, then it will try to unmarshal it as a regular proto message + var cd types.CompressedData + if err := cd.Unmarshal(compressed); err != nil { + // fall back to non-compress unmarshal + return proto.Unmarshal(compressed, target) + } + + // unmarshal was successful, but no data, treat as non-compressed + if cd.Data == nil { + return proto.Unmarshal(compressed, target) + } + + // add other algorithms here if we need to change it + if cd.Algorithm == types.CompressedData_ZLIB { + decompressed, err := decompressZLib(cd.Data) + if err != nil { + return err + } + return proto.Unmarshal(decompressed, target) + } + + return fmt.Errorf("unsupported compression algorithm: %d (%s)", cd.Algorithm, cd.Algorithm.String()) +} diff --git a/utils/compression/compress_test.go b/utils/compression/compress_test.go new file mode 100644 index 000000000..44d733c9a --- /dev/null +++ b/utils/compression/compress_test.go @@ -0,0 +1,242 @@ +package compression + +import ( + "bytes" + "fmt" + "testing" + + "github.com/gogo/protobuf/jsonpb" + "github.com/sei-protocol/sei-chain/x/evm/types" + "github.com/stretchr/testify/require" +) + +type compressFunc func([]byte) ([]byte, error) +type uncompressFunc func([]byte) ([]byte, error) + +func printCompressionRatio(algo string, b, bCompressed []byte) { + originalSize := len(b) + compressedSize := len(bCompressed) + compressionRatio := float64(compressedSize) / float64(originalSize) * 100 + + fmt.Printf("[Debug] (%s) Original size: %d bytes\n", algo, originalSize) + fmt.Printf("[Debug] (%s), compressed size: %d bytes\n", algo, compressedSize) + fmt.Printf("[Debug] (%s), compression ratio: %.2f%%\n", algo, compressionRatio) +} + +const receiptJson = `{ + "tx_type": 0, + "cumulative_gas_used": 0, + "contract_address": null, + "tx_hash_hex": "50000603341192e9af2688fcd052dfbba333d3196d40df121ca8bb92a345a2b4", + "gas_used": 243714, + "effective_gas_price": 1000000000, + "block_number": 12345, + "transaction_index": 0, + "status": 1, + "from": "0x70f67735d4b4d9fcfb3014da2470e2f82a8744c7", + "to": "0x2880ab155794e7179c9ee2e38200202908c17b43", + "vm_error": "", + "logs": [ + { + "address": "0x2880ab155794e7179c9ee2e38200202908c17b43", + "topics": [ + "0xd06a6b7f4918494b3719217d1802786c1f5112a6c1d88fe2cfec00b4584f6aec", + "0x53614f1cb0c031d4af66c04cb9c756234adad0e1cee85303795091499a4084eb" + ], + "data": "00000000000000000000000000000000000000000000000000000000667072e700000000000000000000000000000000000000000000000000000000024c88ca000000000000000000000000000000000000000000000000000000000000d23a", + "index": 0 + } + ], + "logsBloom": "00000400000000000000002000000000000000000000000000000000000000000200000000000000000000000008000000000000000000000000000000000000000000000000000000400000000010000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000010001400000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000100008000000000800000000000" +}` + +func TestBackwardsCompatibility(t *testing.T) { + var r types.Receipt + err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r) + require.NoError(t, err) + receiptBytes, err := r.Marshal() + require.NoError(t, err) + + type compressTest struct { + Name string + Stored func() []byte + Expected []byte + WantErr bool + } + + for _, test := range []compressTest{ + { + Name: "not-compressed", + Stored: func() []byte { + return receiptBytes + }, + Expected: receiptBytes, + }, + { + Name: "compressed", + Stored: func() []byte { + cb, err := CompressMessage(&r) + require.NoError(t, err) + return cb + }, + Expected: receiptBytes, + }, + { + Name: "bad data", + Stored: func() []byte { + // not a receipt + block := &types.Log{ + Address: "0x1230ab155794e7179c9ee2e38200202908c17b43", + } + b, err := block.Marshal() + require.NoError(t, err) + return b + }, + WantErr: true, + }, + } { + t.Run(test.Name, func(t *testing.T) { + var result types.Receipt + err := DecompressMessage(&result, test.Stored()) + if test.WantErr { + require.Error(t, err) + } else { + // should have same bytes as expected bytes + require.NoError(t, err) + resultBytes, err := result.Marshal() + require.NoError(t, err) + require.Equal(t, test.Expected, resultBytes) + } + }) + } +} + +func TestCompressMessage(t *testing.T) { + var r types.Receipt + err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r) + require.NoError(t, err) + + raw, err := r.Marshal() + require.NoError(t, err) + + compressed, err := CompressMessage(&r) + require.NoError(t, err) + + var r2 types.Receipt + err = DecompressMessage(&r2, compressed) + require.NoError(t, err) + + require.Equal(t, r, r2) + printCompressionRatio("zlib", raw, compressed) +} + +func TestCompressionRatio(t *testing.T) { + r := types.Receipt{} + err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r) + require.NoError(t, err) + + b, err := r.Marshal() + require.NoError(t, err) + + tests := []struct { + Name string + Compress compressFunc + Uncompress uncompressFunc + }{ + //{ + // Name: "brotli", + // Compress: compressBrotli, + // Uncompress: decompressBrotli, + //}, + //{ + // Name: "bzip2", + // Compress: compressBzip2, + // Uncompress: decompressBzip2, + //}, + //{ + // Name: "lzma", + // Compress: compressLzma, + // Uncompress: decompressLzma, + //}, + //{ + // Name: "lz4", + // Compress: compressLz4, + // Uncompress: decompressLz4, + //}, + { + Name: "zlib", + Compress: compressZLib, + Uncompress: decompressZLib, + }, + //{ + // Name: "zstd", + // Compress: compressZstd, + // Uncompress: decompressZstd, + //}, + //{ + // Name: "snappy", + // Compress: compressSnappy, + // Uncompress: decompressSnappy, + //}, + //{ + // Name: "gzip", + // Compress: compressGzip, + // Uncompress: decompressGzip, + //}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + bCompressed, err := test.Compress(b) + require.NoError(t, err) + + bUncompressed, err := test.Uncompress(bCompressed) + require.NoError(t, err) + + require.Equal(t, string(b), string(bUncompressed), "%s: expected %s but got %s", test.Name, b, bUncompressed) + printCompressionRatio(test.Name, b, bCompressed) + }) + } +} + +func benchmarkCompression(b *testing.B, name string, compress compressFunc, uncompress uncompressFunc) { + r := types.Receipt{} + err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r) + require.NoError(b, err) + + data, err := r.Marshal() + require.NoError(b, err) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + compressedData, err := compress(data) + require.NoError(b, err) + + _, err = uncompress(compressedData) + require.NoError(b, err) + } +} + +func BenchmarkCompression(b *testing.B) { + benchmarks := []struct { + name string + compress compressFunc + uncompress uncompressFunc + }{ + //{"brotli", compressBrotli, decompressBrotli}, + //{"bzip2", compressBzip2, decompressBzip2}, + //{"lzma", compressLzma, decompressLzma}, + {"zlib", compressZLib, decompressZLib}, + //{"lz4", compressLz4, decompressLz4}, + //{"zstd", compressZstd, decompressZstd}, + //{"snappy", compressSnappy, decompressSnappy}, + //{"gzip", compressGzip, decompressGzip}, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + benchmarkCompression(b, bm.name, bm.compress, bm.uncompress) + }) + } +} diff --git a/utils/compression/zlib.go b/utils/compression/zlib.go new file mode 100644 index 000000000..f1a5371c4 --- /dev/null +++ b/utils/compression/zlib.go @@ -0,0 +1,34 @@ +package compression + +import ( + "bytes" + + "compress/zlib" +) + +func compressZLib(b []byte) ([]byte, error) { + var buf bytes.Buffer + writer := zlib.NewWriter(&buf) + _, err := writer.Write(b) + if err != nil { + return nil, err + } + writer.Close() + + return buf.Bytes(), nil +} + +func decompressZLib(b []byte) ([]byte, error) { + r, err := zlib.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer r.Close() + + var buf bytes.Buffer + if _, err := buf.ReadFrom(r); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/x/evm/keeper/receipt.go b/x/evm/keeper/receipt.go index 43e6c9388..c1001c4af 100644 --- a/x/evm/keeper/receipt.go +++ b/x/evm/keeper/receipt.go @@ -2,9 +2,9 @@ package keeper import ( "errors" - sdk "github.com/cosmos/cosmos-sdk/types" "github.com/ethereum/go-ethereum/common" + "github.com/sei-protocol/sei-chain/utils/compression" "github.com/sei-protocol/sei-chain/x/evm/types" ) @@ -17,8 +17,9 @@ func (k *Keeper) GetReceipt(ctx sdk.Context, txHash common.Hash) (*types.Receipt if bz == nil { return nil, errors.New("not found") } - r := types.Receipt{} - if err := r.Unmarshal(bz); err != nil { + + var r types.Receipt + if err := compression.DecompressMessage(&r, bz); err != nil { return nil, err } return &r, nil @@ -26,7 +27,7 @@ func (k *Keeper) GetReceipt(ctx sdk.Context, txHash common.Hash) (*types.Receipt func (k *Keeper) SetReceipt(ctx sdk.Context, txHash common.Hash, receipt *types.Receipt) error { store := ctx.KVStore(k.storeKey) - bz, err := receipt.Marshal() + bz, err := compression.CompressMessage(receipt) if err != nil { return err } From 0afd5c3171f661f81f2ea27c48a188f09738331d Mon Sep 17 00:00:00 2001 From: Steven Landers Date: Mon, 17 Jun 2024 16:50:20 -0400 Subject: [PATCH 2/6] remove excess libs --- go.mod | 8 ++------ go.sum | 11 ----------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index c6e224543..ce067a213 100644 --- a/go.mod +++ b/go.mod @@ -6,20 +6,17 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/CosmWasm/wasmd v0.27.0 github.com/CosmWasm/wasmvm v1.5.2 - github.com/andybalholm/brotli v1.1.0 github.com/armon/go-metrics v0.4.1 github.com/btcsuite/btcd v0.22.1 github.com/cosmos/cosmos-sdk v0.45.10 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583 github.com/cosmos/ibc-go/v3 v3.0.0 - github.com/dsnet/compress v0.0.1 github.com/ethereum/go-ethereum v1.13.2 github.com/go-playground/validator/v10 v10.4.1 github.com/gogo/protobuf v1.3.3 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/golang/protobuf v1.5.4 - github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/golangci/golangci-lint v1.46.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 @@ -27,9 +24,7 @@ require ( github.com/holiman/uint256 v1.2.4 github.com/justinas/alice v1.2.0 github.com/k0kubun/pp/v3 v3.2.0 - github.com/klauspost/compress v1.16.3 github.com/mitchellh/mapstructure v1.5.0 - github.com/pierrec/lz4/v4 v4.1.21 github.com/pkg/errors v0.9.1 github.com/rakyll/statik v0.1.7 github.com/rs/cors v1.8.2 @@ -43,7 +38,6 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d github.com/tendermint/tendermint v0.37.0-dev github.com/tendermint/tm-db v0.6.8-0.20220519162814-e24b96538a12 - github.com/ulikunitz/xz v0.5.6 go.opentelemetry.io/otel v1.9.0 go.opentelemetry.io/otel/trace v1.9.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa @@ -155,6 +149,7 @@ require ( github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect @@ -203,6 +198,7 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kisielk/errcheck v1.6.0 // indirect github.com/kisielk/gotool v1.0.0 // indirect + github.com/klauspost/compress v1.16.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kulti/thelper v0.6.2 // indirect diff --git a/go.sum b/go.sum index e8c82ace7..48d365910 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,6 @@ github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= @@ -388,9 +386,6 @@ github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRP github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -926,7 +921,6 @@ github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -935,7 +929,6 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1214,8 +1207,6 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= @@ -1523,8 +1514,6 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.5 h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqzi/CzI= From f7f07a8b555866b3d6ddda17a5e437acf7a99175 Mon Sep 17 00:00:00 2001 From: Steven Landers Date: Mon, 17 Jun 2024 16:52:10 -0400 Subject: [PATCH 3/6] cleanup import --- utils/compression/compress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/compression/compress.go b/utils/compression/compress.go index 0021684f0..5f9abd486 100644 --- a/utils/compression/compress.go +++ b/utils/compression/compress.go @@ -2,6 +2,7 @@ package compression import ( "fmt" + "github.com/gogo/protobuf/proto" "github.com/sei-protocol/sei-chain/x/evm/types" ) From c5a94136b51a3432f50cd44a463ad2c5f2a445c9 Mon Sep 17 00:00:00 2001 From: Steven Landers Date: Mon, 17 Jun 2024 16:53:55 -0400 Subject: [PATCH 4/6] cleanup --- go.mod | 2 + go.sum | 3 + proto/evm/compression.proto | 1 - utils/compression/compress.go | 2 +- x/evm/types/compression.pb.go | 382 ++++++++++++++++++++++++++++++++++ 5 files changed, 388 insertions(+), 2 deletions(-) create mode 100644 x/evm/types/compression.pb.go diff --git a/go.mod b/go.mod index ce067a213..46299debc 100644 --- a/go.mod +++ b/go.mod @@ -126,6 +126,7 @@ require ( github.com/fzipp/gocyclo v0.5.1 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect + github.com/ghodss/yaml v1.0.0 // indirect github.com/go-critic/go-critic v0.6.3 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -173,6 +174,7 @@ require ( github.com/gostaticanalysis/nilerr v0.1.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect diff --git a/go.sum b/go.sum index 48d365910..83e617233 100644 --- a/go.sum +++ b/go.sum @@ -457,6 +457,7 @@ github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2Gihuqh github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -765,6 +766,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/guptarohit/asciigraph v0.5.5/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag= diff --git a/proto/evm/compression.proto b/proto/evm/compression.proto index ced18b90a..c9aeda331 100644 --- a/proto/evm/compression.proto +++ b/proto/evm/compression.proto @@ -12,7 +12,6 @@ message CompressedData { ZLIB = 0; } - // if true, it informs the client that the data is compressed bytes data = 1; Algorithm algorithm = 2; } \ No newline at end of file diff --git a/utils/compression/compress.go b/utils/compression/compress.go index 5f9abd486..a8d73f9c1 100644 --- a/utils/compression/compress.go +++ b/utils/compression/compress.go @@ -2,7 +2,7 @@ package compression import ( "fmt" - + "github.com/gogo/protobuf/proto" "github.com/sei-protocol/sei-chain/x/evm/types" ) diff --git a/x/evm/types/compression.pb.go b/x/evm/types/compression.pb.go new file mode 100644 index 000000000..90e0f2ed1 --- /dev/null +++ b/x/evm/types/compression.pb.go @@ -0,0 +1,382 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: evm/compression.proto + +package types + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type CompressedData_Algorithm int32 + +const ( + CompressedData_ZLIB CompressedData_Algorithm = 0 +) + +var CompressedData_Algorithm_name = map[int32]string{ + 0: "ZLIB", +} + +var CompressedData_Algorithm_value = map[string]int32{ + "ZLIB": 0, +} + +func (x CompressedData_Algorithm) String() string { + return proto.EnumName(CompressedData_Algorithm_name, int32(x)) +} + +func (CompressedData_Algorithm) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_e7e00623403436c4, []int{0, 0} +} + +// CompressedData is only used for receipts as the moment +// It is not meant to be used outside of storage and retrieval +type CompressedData struct { + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Algorithm CompressedData_Algorithm `protobuf:"varint,2,opt,name=algorithm,proto3,enum=seiprotocol.seichain.evm.CompressedData_Algorithm" json:"algorithm,omitempty"` +} + +func (m *CompressedData) Reset() { *m = CompressedData{} } +func (m *CompressedData) String() string { return proto.CompactTextString(m) } +func (*CompressedData) ProtoMessage() {} +func (*CompressedData) Descriptor() ([]byte, []int) { + return fileDescriptor_e7e00623403436c4, []int{0} +} +func (m *CompressedData) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CompressedData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CompressedData.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CompressedData) XXX_Merge(src proto.Message) { + xxx_messageInfo_CompressedData.Merge(m, src) +} +func (m *CompressedData) XXX_Size() int { + return m.Size() +} +func (m *CompressedData) XXX_DiscardUnknown() { + xxx_messageInfo_CompressedData.DiscardUnknown(m) +} + +var xxx_messageInfo_CompressedData proto.InternalMessageInfo + +func (m *CompressedData) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *CompressedData) GetAlgorithm() CompressedData_Algorithm { + if m != nil { + return m.Algorithm + } + return CompressedData_ZLIB +} + +func init() { + proto.RegisterEnum("seiprotocol.seichain.evm.CompressedData_Algorithm", CompressedData_Algorithm_name, CompressedData_Algorithm_value) + proto.RegisterType((*CompressedData)(nil), "seiprotocol.seichain.evm.CompressedData") +} + +func init() { proto.RegisterFile("evm/compression.proto", fileDescriptor_e7e00623403436c4) } + +var fileDescriptor_e7e00623403436c4 = []byte{ + // 230 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4d, 0x2d, 0xcb, 0xd5, + 0x4f, 0xce, 0xcf, 0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0xce, 0xcc, 0xcf, 0xd3, 0x2b, 0x28, 0xca, 0x2f, + 0xc9, 0x17, 0x92, 0x28, 0x4e, 0xcd, 0x04, 0xb3, 0x92, 0xf3, 0x73, 0xf4, 0x8a, 0x53, 0x33, 0x93, + 0x33, 0x12, 0x33, 0xf3, 0xf4, 0x52, 0xcb, 0x72, 0xa5, 0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0x52, + 0xfa, 0x20, 0x16, 0x44, 0xbd, 0x52, 0x2f, 0x23, 0x17, 0x9f, 0x33, 0xd4, 0x94, 0xd4, 0x14, 0x97, + 0xc4, 0x92, 0x44, 0x21, 0x21, 0x2e, 0x96, 0x94, 0xc4, 0x92, 0x44, 0x09, 0x46, 0x05, 0x46, 0x0d, + 0x9e, 0x20, 0x30, 0x5b, 0x28, 0x80, 0x8b, 0x33, 0x31, 0x27, 0x3d, 0xbf, 0x28, 0xb3, 0x24, 0x23, + 0x57, 0x82, 0x49, 0x81, 0x51, 0x83, 0xcf, 0xc8, 0x48, 0x0f, 0x97, 0x55, 0x7a, 0xa8, 0x06, 0xea, + 0x39, 0xc2, 0x74, 0x06, 0x21, 0x0c, 0x51, 0x12, 0xe5, 0xe2, 0x84, 0x8b, 0x0b, 0x71, 0x70, 0xb1, + 0x44, 0xf9, 0x78, 0x3a, 0x09, 0x30, 0x38, 0xb9, 0x9f, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, + 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x5c, 0x78, 0x2c, 0xc7, 0x70, 0xe3, 0xb1, + 0x1c, 0x43, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, 0xae, 0x7e, 0x71, + 0x6a, 0xa6, 0x2e, 0xcc, 0x6a, 0x30, 0x07, 0x6c, 0xb7, 0x7e, 0x85, 0x3e, 0x28, 0x50, 0x4a, 0x2a, + 0x0b, 0x52, 0x8b, 0x93, 0xd8, 0xc0, 0xf2, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x07, 0xfc, + 0x22, 0x24, 0x28, 0x01, 0x00, 0x00, +} + +func (m *CompressedData) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CompressedData) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CompressedData) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Algorithm != 0 { + i = encodeVarintCompression(dAtA, i, uint64(m.Algorithm)) + i-- + dAtA[i] = 0x10 + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = encodeVarintCompression(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintCompression(dAtA []byte, offset int, v uint64) int { + offset -= sovCompression(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *CompressedData) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + sovCompression(uint64(l)) + } + if m.Algorithm != 0 { + n += 1 + sovCompression(uint64(m.Algorithm)) + } + return n +} + +func sovCompression(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozCompression(x uint64) (n int) { + return sovCompression(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *CompressedData) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCompression + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CompressedData: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CompressedData: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCompression + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthCompression + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthCompression + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Algorithm", wireType) + } + m.Algorithm = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCompression + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Algorithm |= CompressedData_Algorithm(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipCompression(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCompression + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipCompression(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCompression + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCompression + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCompression + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthCompression + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupCompression + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthCompression + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthCompression = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowCompression = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupCompression = fmt.Errorf("proto: unexpected end of group") +) From 232b8ae6d0e2dfb8b8c1f7ab80dedfd33002d8fa Mon Sep 17 00:00:00 2001 From: Steven Landers Date: Mon, 17 Jun 2024 16:57:55 -0400 Subject: [PATCH 5/6] cleanup --- Makefile | 2 +- x/evm/keeper/receipt.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7706706ca..3860005eb 100644 --- a/Makefile +++ b/Makefile @@ -223,4 +223,4 @@ test-group-%:split-test-packages .PHONY: proto proto: - ignite generate proto-go -y \ No newline at end of file + ignite generate proto-go -y diff --git a/x/evm/keeper/receipt.go b/x/evm/keeper/receipt.go index c1001c4af..772c61bfc 100644 --- a/x/evm/keeper/receipt.go +++ b/x/evm/keeper/receipt.go @@ -2,6 +2,7 @@ package keeper import ( "errors" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/ethereum/go-ethereum/common" "github.com/sei-protocol/sei-chain/utils/compression" From f3bd27439bd79bf96d855a05102b457fc64477f6 Mon Sep 17 00:00:00 2001 From: Steven Landers Date: Mon, 17 Jun 2024 21:30:36 -0400 Subject: [PATCH 6/6] switch to snappy --- proto/evm/compression.proto | 2 +- utils/compression/compress.go | 8 ++++---- utils/compression/compress_test.go | 12 ++++++------ utils/compression/snappy.go | 30 ++++++++++++++++++++++++++++++ x/evm/types/compression.pb.go | 26 +++++++++++++------------- 5 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 utils/compression/snappy.go diff --git a/proto/evm/compression.proto b/proto/evm/compression.proto index c9aeda331..d2131d505 100644 --- a/proto/evm/compression.proto +++ b/proto/evm/compression.proto @@ -9,7 +9,7 @@ option go_package = "github.com/sei-protocol/sei-chain/x/evm/types"; // It is not meant to be used outside of storage and retrieval message CompressedData { enum Algorithm { - ZLIB = 0; + SNAPPY = 0; } bytes data = 1; diff --git a/utils/compression/compress.go b/utils/compression/compress.go index a8d73f9c1..039e3282f 100644 --- a/utils/compression/compress.go +++ b/utils/compression/compress.go @@ -13,14 +13,14 @@ func CompressMessage(message proto.Message) ([]byte, error) { return nil, err } - bCompressed, err := compressZLib(b) + bCompressed, err := compressSnappy(b) if err != nil { return nil, err } cd := &types.CompressedData{ Data: bCompressed, - Algorithm: types.CompressedData_ZLIB, + Algorithm: types.CompressedData_SNAPPY, } return cd.Marshal() } @@ -40,8 +40,8 @@ func DecompressMessage(target proto.Message, compressed []byte) error { } // add other algorithms here if we need to change it - if cd.Algorithm == types.CompressedData_ZLIB { - decompressed, err := decompressZLib(cd.Data) + if cd.Algorithm == types.CompressedData_SNAPPY { + decompressed, err := decompressSnappy(cd.Data) if err != nil { return err } diff --git a/utils/compression/compress_test.go b/utils/compression/compress_test.go index 44d733c9a..094160422 100644 --- a/utils/compression/compress_test.go +++ b/utils/compression/compress_test.go @@ -173,11 +173,11 @@ func TestCompressionRatio(t *testing.T) { // Compress: compressZstd, // Uncompress: decompressZstd, //}, - //{ - // Name: "snappy", - // Compress: compressSnappy, - // Uncompress: decompressSnappy, - //}, + { + Name: "snappy", + Compress: compressSnappy, + Uncompress: decompressSnappy, + }, //{ // Name: "gzip", // Compress: compressGzip, @@ -230,7 +230,7 @@ func BenchmarkCompression(b *testing.B) { {"zlib", compressZLib, decompressZLib}, //{"lz4", compressLz4, decompressLz4}, //{"zstd", compressZstd, decompressZstd}, - //{"snappy", compressSnappy, decompressSnappy}, + {"snappy", compressSnappy, decompressSnappy}, //{"gzip", compressGzip, decompressGzip}, } diff --git a/utils/compression/snappy.go b/utils/compression/snappy.go new file mode 100644 index 000000000..da38be651 --- /dev/null +++ b/utils/compression/snappy.go @@ -0,0 +1,30 @@ +package compression + +import ( + "bytes" + + "github.com/golang/snappy" +) + +func compressSnappy(b []byte) ([]byte, error) { + var buf bytes.Buffer + writer := snappy.NewBufferedWriter(&buf) + _, err := writer.Write(b) + if err != nil { + return nil, err + } + writer.Close() + + return buf.Bytes(), nil +} + +func decompressSnappy(b []byte) ([]byte, error) { + r := snappy.NewReader(bytes.NewReader(b)) + + var buf bytes.Buffer + if _, err := buf.ReadFrom(r); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/x/evm/types/compression.pb.go b/x/evm/types/compression.pb.go index 90e0f2ed1..f5d09ac24 100644 --- a/x/evm/types/compression.pb.go +++ b/x/evm/types/compression.pb.go @@ -26,15 +26,15 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type CompressedData_Algorithm int32 const ( - CompressedData_ZLIB CompressedData_Algorithm = 0 + CompressedData_SNAPPY CompressedData_Algorithm = 0 ) var CompressedData_Algorithm_name = map[int32]string{ - 0: "ZLIB", + 0: "SNAPPY", } var CompressedData_Algorithm_value = map[string]int32{ - "ZLIB": 0, + "SNAPPY": 0, } func (x CompressedData_Algorithm) String() string { @@ -96,7 +96,7 @@ func (m *CompressedData) GetAlgorithm() CompressedData_Algorithm { if m != nil { return m.Algorithm } - return CompressedData_ZLIB + return CompressedData_SNAPPY } func init() { @@ -107,22 +107,22 @@ func init() { func init() { proto.RegisterFile("evm/compression.proto", fileDescriptor_e7e00623403436c4) } var fileDescriptor_e7e00623403436c4 = []byte{ - // 230 bytes of a gzipped FileDescriptorProto + // 232 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4d, 0x2d, 0xcb, 0xd5, 0x4f, 0xce, 0xcf, 0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0xce, 0xcc, 0xcf, 0xd3, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x92, 0x28, 0x4e, 0xcd, 0x04, 0xb3, 0x92, 0xf3, 0x73, 0xf4, 0x8a, 0x53, 0x33, 0x93, 0x33, 0x12, 0x33, 0xf3, 0xf4, 0x52, 0xcb, 0x72, 0xa5, 0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0x52, - 0xfa, 0x20, 0x16, 0x44, 0xbd, 0x52, 0x2f, 0x23, 0x17, 0x9f, 0x33, 0xd4, 0x94, 0xd4, 0x14, 0x97, + 0xfa, 0x20, 0x16, 0x44, 0xbd, 0x52, 0x3f, 0x23, 0x17, 0x9f, 0x33, 0xd4, 0x94, 0xd4, 0x14, 0x97, 0xc4, 0x92, 0x44, 0x21, 0x21, 0x2e, 0x96, 0x94, 0xc4, 0x92, 0x44, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x30, 0x5b, 0x28, 0x80, 0x8b, 0x33, 0x31, 0x27, 0x3d, 0xbf, 0x28, 0xb3, 0x24, 0x23, 0x57, 0x82, 0x49, 0x81, 0x51, 0x83, 0xcf, 0xc8, 0x48, 0x0f, 0x97, 0x55, 0x7a, 0xa8, 0x06, 0xea, - 0x39, 0xc2, 0x74, 0x06, 0x21, 0x0c, 0x51, 0x12, 0xe5, 0xe2, 0x84, 0x8b, 0x0b, 0x71, 0x70, 0xb1, - 0x44, 0xf9, 0x78, 0x3a, 0x09, 0x30, 0x38, 0xb9, 0x9f, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, - 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x5c, 0x78, 0x2c, 0xc7, 0x70, 0xe3, 0xb1, - 0x1c, 0x43, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, 0xae, 0x7e, 0x71, - 0x6a, 0xa6, 0x2e, 0xcc, 0x6a, 0x30, 0x07, 0x6c, 0xb7, 0x7e, 0x85, 0x3e, 0x28, 0x50, 0x4a, 0x2a, - 0x0b, 0x52, 0x8b, 0x93, 0xd8, 0xc0, 0xf2, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x07, 0xfc, - 0x22, 0x24, 0x28, 0x01, 0x00, 0x00, + 0x39, 0xc2, 0x74, 0x06, 0x21, 0x0c, 0x51, 0x12, 0xe7, 0xe2, 0x84, 0x8b, 0x0b, 0x71, 0x71, 0xb1, + 0x05, 0xfb, 0x39, 0x06, 0x04, 0x44, 0x0a, 0x30, 0x38, 0xb9, 0x9f, 0x78, 0x24, 0xc7, 0x78, 0xe1, + 0x91, 0x1c, 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x5c, 0x78, 0x2c, 0xc7, 0x70, + 0xe3, 0xb1, 0x1c, 0x43, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, 0xae, + 0x7e, 0x71, 0x6a, 0xa6, 0x2e, 0xcc, 0x72, 0x30, 0x07, 0x6c, 0xbb, 0x7e, 0x85, 0x3e, 0x28, 0x58, + 0x4a, 0x2a, 0x0b, 0x52, 0x8b, 0x93, 0xd8, 0xc0, 0xf2, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xdf, 0x44, 0xa1, 0xb6, 0x2a, 0x01, 0x00, 0x00, } func (m *CompressedData) Marshal() (dAtA []byte, err error) {