diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d8812c2624..c77eac5df71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,55 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +## Fixed +- [2320](https://github.com/FuelLabs/fuel-core/issues/2320): Prevent `/health` and `/v1/health` from being throttled by the concurrency limiter. + +## [Version 0.38.0] + ### Added +- [2309](https://github.com/FuelLabs/fuel-core/pull/2309): Limit number of concurrent queries to the graphql service. +- [2216](https://github.com/FuelLabs/fuel-core/pull/2216): Add more function to the state and task of TxPoolV2 to handle the future interactions with others modules (PoA, BlockProducer, BlockImporter and P2P). +- [2263](https://github.com/FuelLabs/fuel-core/pull/2263): Transaction pool is now included in all modules of the code it has requires modifications on different modules : + - The PoA is now notify only when there is new transaction and not using the `tx_update_sender` anymore. + - The Pool transaction source for the executor is now locking the pool until the block production is finished. + - Reading operations on the pool is now asynchronous and it’s the less prioritized operation on the Pool, API has been updated accordingly. + - GasPrice is no more using async to allow the transactions verifications to not use async anymore + + We also added a lot of new configuration cli parameters to fine-tune TxPool configuration. + This PR also changes the way we are making the heavy work processor and a sync and asynchronous version is available in services folder (usable by anyone) + P2P now use separate heavy work processor for DB and TxPool interactions. + +### Removed +- [2306](https://github.com/FuelLabs/fuel-core/pull/2306): Removed hack for genesis asset contract from the code. + +## [Version 0.37.1] + +### Fixed +- [2304](https://github.com/FuelLabs/fuel-core/pull/2304): Add initialization for the genesis base asset contract. + +## [Version 0.37.0] + +### Added +- [1609](https://github.com/FuelLabs/fuel-core/pull/1609): Add DA compression support. Compressed blocks are stored in the offchain database when blocks are produced, and can be fetched using the GraphQL API. +- [2290](https://github.com/FuelLabs/fuel-core/pull/2290): Added a new CLI argument `--graphql-max-directives`. The default value is `10`. - [2195](https://github.com/FuelLabs/fuel-core/pull/2195): Added enforcement of the limit on the size of the L2 transactions per block according to the `block_transaction_size_limit` parameter. - [2131](https://github.com/FuelLabs/fuel-core/pull/2131): Add flow in TxPool in order to ask to newly connected peers to share their transaction pool - [2182](https://github.com/FuelLabs/fuel-core/pull/2151): Limit number of transactions that can be fetched via TxSource::next - [2189](https://github.com/FuelLabs/fuel-core/pull/2151): Select next DA height to never include more than u16::MAX -1 transactions from L1. - +- [2265](https://github.com/FuelLabs/fuel-core/pull/2265): Integrate Block Committer API for DA Block Costs. +- [2162](https://github.com/FuelLabs/fuel-core/pull/2162): Pool structure with dependencies, etc.. for the next transaction pool module. Also adds insertion/verification process in PoolV2 and tests refactoring +- [2280](https://github.com/FuelLabs/fuel-core/pull/2280): Allow comma separated relayer addresses in cli +- [2299](https://github.com/FuelLabs/fuel-core/pull/2299): Support blobs in the predicates. +- [2300](https://github.com/FuelLabs/fuel-core/pull/2300): Added new function to `fuel-core-client` for checking whether a blob exists. ### Changed - [2233]((https://github.com/FuelLabs/fuel-core/pull/2233): New changes to HistoricalRocksDB are written to ModificationsHistoryV2 #### Breaking +- [2299](https://github.com/FuelLabs/fuel-core/pull/2299): Anyone who wants to participate in the transaction broadcasting via p2p must upgrade to support new predicates on the TxPool level. +- [2299](https://github.com/FuelLabs/fuel-core/pull/2299): Upgraded `fuel-vm` to `0.58.0`. More information in the [release](https://github.com/FuelLabs/fuel-vm/releases/tag/v0.58.0). +- [2276](https://github.com/FuelLabs/fuel-core/pull/2276): Changed how complexity for blocks is calculated. The default complexity now is 80_000. All queries that somehow touch the block header now are more expensive. +- [2290](https://github.com/FuelLabs/fuel-core/pull/2290): Added a new GraphQL limit on number of `directives`. The default value is `10`. - [2206](https://github.com/FuelLabs/fuel-core/pull/2206): Use timestamp of last block when dry running transactions. - [2153](https://github.com/FuelLabs/fuel-core/pull/2153): Updated default gas costs for the local testnet configuration to match `fuel-core 0.35.0`. diff --git a/Cargo.lock b/Cargo.lock index 0b4f1eae4d7..41fe878f550 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,11 +14,11 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ - "gimli 0.31.0", + "gimli 0.31.1", ] [[package]] @@ -353,7 +353,7 @@ dependencies = [ "futures-timer", "futures-util", "http 1.1.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "mime", "multer", "num-traits", @@ -405,7 +405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" dependencies = [ "bytes", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_json", ] @@ -543,9 +543,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -554,9 +554,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", @@ -673,9 +673,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.7" +version = "1.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +checksum = "7198e6f03240fdceba36656d8be440297b6b82270325908c7381f37d826a74f6" dependencies = [ "aws-credential-types", "aws-runtime", @@ -740,9 +740,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.45.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0caf20b8855dbeb458552e6c8f8f9eb92b95e4a131725b93540ec73d60c38eb3" +checksum = "e33590e8d45206fdc4273ded8a1f292bcceaadd513037aa790fc67b237bc30ee" dependencies = [ "aws-credential-types", "aws-runtime", @@ -762,9 +762,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.44.0" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +checksum = "e33ae899566f3d395cbf42858e433930682cc9c1889fa89318896082fef45efb" dependencies = [ "aws-credential-types", "aws-runtime", @@ -784,9 +784,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.45.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +checksum = "f39c09e199ebd96b9f860b0fce4b6625f211e064ad7c8693b72ecf7ef03881e0" dependencies = [ "aws-credential-types", "aws-runtime", @@ -806,9 +806,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.44.0" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +checksum = "3d95f93a98130389eb6233b9d615249e543f6c24a68ca1f109af9ca5164a8765" dependencies = [ "aws-credential-types", "aws-runtime", @@ -902,9 +902,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +checksum = "a065c0fe6fdbdf9f11817eb68582b2ab4aff9e9c39e986ae48f7ec576c6322db" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1019,7 +1019,7 @@ dependencies = [ "sync_wrapper", "tokio", "tower", - "tower-http", + "tower-http 0.3.5", "tower-layer", "tower-service", ] @@ -1228,7 +1228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", - "regex-automata 0.4.7", + "regex-automata 0.4.8", "serde", ] @@ -1330,9 +1330,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.22" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "jobserver", "libc", @@ -1465,9 +1465,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive 4.5.18", @@ -1475,9 +1475,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -1626,9 +1626,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" +checksum = "0121754e84117e65f9d90648ee6aa4882a6e63110307ab73967a4c5e7e69e586" dependencies = [ "cfg-if", "cpufeatures", @@ -1777,18 +1777,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305d51c180ebdc46ef61bc60c54ae6512db3bc9a05842a1f1e762e45977019ab" +checksum = "4a41b85213deedf877555a7878ca9fb680ccba8183611c4bb8030ed281b2ad83" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3247afacd9b13d620033f3190d9e49d1beefc1acb33d5604a249956c9c13709" +checksum = "690d8ae6c73748e5ce3d8fe59034dceadb8823e6c8994ba324141c5eae909b0e" dependencies = [ "serde", "serde_derive", @@ -1796,9 +1796,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7ca95e831c18d1356da783765c344207cbdffea91e13e47fa9327dbb2e0719" +checksum = "0ce027a7b16f8b86f60ff6819615273635186d607a0c225ee6ac340d7d18f978" dependencies = [ "bumpalo", "cranelift-bforest", @@ -1819,33 +1819,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "450c105fa1e51bfba4e95a86e926504a867ad5639d63f31d43fe3b7ec1f1c9ef" +checksum = "f0a2d2ab65e6cbf91f81781d8da65ec2005510f18300eff21a99526ed6785863" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479117cd1266881479908d383086561cee37e49affbea9b1e6b594cc21cc220" +checksum = "efcff860573cf3db9ae98fbd949240d78b319df686cc306872e7fab60e9c84d7" [[package]] name = "cranelift-control" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34378804f0abfdd22c068a741cfeed86938b92375b2a96fb0b42c878e0141bfb" +checksum = "69d70e5b75c2d5541ef80a99966ccd97aaa54d2a6af19ea31759a28538e1685a" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a48cb0a194c9ba82fec35a1e492055388d89b2e3c03dee9dcf2488892be8004d" +checksum = "d21d3089714278920030321829090d9482c91e5ff2339f2f697f8425bffdcba3" dependencies = [ "cranelift-bitset", "serde", @@ -1854,9 +1854,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8327afc6c1c05f4be62fefce5b439fa83521c65363a322e86ea32c85e7ceaf64" +checksum = "7308482930f2a2fad4fe25a06054f6f9a4ee1ab97264308c661b037cb60001a3" dependencies = [ "cranelift-codegen", "log", @@ -1866,15 +1866,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56b08621c00321efcfa3eee6a3179adc009e21ea8d24ca7adc3c326184bc3f48" +checksum = "ab4c59e259dab0e6958dabcc536b30845574f027ba6e5000498cdaf7e7ed2d30" [[package]] name = "cranelift-native" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51180b147c8557c1196c77b098f04140c91962e135ea152cd2fcabf40cf365c" +checksum = "d77ac3dfb61ef3159998105116acdfeaec75e4296c43ee2dcc4ea39838c0080e" dependencies = [ "cranelift-codegen", "libc", @@ -1883,9 +1883,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.110.2" +version = "0.110.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019e3dccb7f15e0bc14f0ddc034ec608a66df8e05c9e1e16f75a7716f8461799" +checksum = "1d883f1b8d3d1dab4797407117bc8a1824f4a1fe86654aee2ee3205613f77d3e" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -1915,7 +1915,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.18", + "clap 4.5.20", "criterion-plot", "futures", "is-terminal", @@ -2984,9 +2984,9 @@ dependencies = [ [[package]] name = "eventsource-client" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c80c6714d1a380314fcb11a22eeff022e1e1c9642f0bb54e15dc9cb29f37b29" +checksum = "43ddc25e1ad2cc0106d5e2d967397b4fb2068a66677ee9b0eea4600e5cfe8fb4" dependencies = [ "futures", "hyper", @@ -3100,6 +3100,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -3139,30 +3145,42 @@ dependencies = [ [[package]] name = "fuel-asm" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b83807968cd62babe6cd70c94b76c86f302c8573da18b3c69135688eceecd" +checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" dependencies = [ "bitflags 2.6.0", - "fuel-types 0.57.1", + "fuel-types 0.58.2", "serde", "strum 0.24.1", ] +[[package]] +name = "fuel-compression" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e42841f56f76ed759b3f516e5188d5c42de47015bee951651660c13b6dfa6c" +dependencies = [ + "fuel-derive 0.58.2", + "fuel-types 0.58.2", + "serde", +] + [[package]] name = "fuel-core" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "assert_matches", "async-graphql", "async-trait", "axum", - "clap 4.5.18", + "clap 4.5.20", "derive_more", "enum-iterator", "fuel-core", "fuel-core-chain-config", + "fuel-core-compression", "fuel-core-consensus-module", "fuel-core-database", "fuel-core-executor", @@ -3178,7 +3196,7 @@ dependencies = [ "fuel-core-sync", "fuel-core-trace", "fuel-core-txpool", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "fuel-core-upgradable-executor", "futures", "hex", @@ -3188,6 +3206,7 @@ dependencies = [ "mockall", "num_cpus", "parking_lot", + "paste", "postcard", "proptest", "rand", @@ -3204,7 +3223,8 @@ dependencies = [ "tokio-rayon", "tokio-stream", "tokio-util", - "tower-http", + "tower", + "tower-http 0.4.4", "tracing", "uuid 1.10.0", ] @@ -3215,7 +3235,7 @@ version = "0.0.0" dependencies = [ "anyhow", "async-trait", - "clap 4.5.18", + "clap 4.5.20", "criterion", "ctrlc", "ed25519-dalek", @@ -3227,7 +3247,7 @@ dependencies = [ "fuel-core-services", "fuel-core-storage", "fuel-core-sync", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "itertools 0.12.1", "num_enum", @@ -3248,24 +3268,25 @@ dependencies = [ [[package]] name = "fuel-core-bft" -version = "0.36.0" +version = "0.38.0" [[package]] name = "fuel-core-bin" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "aws-config", "aws-sdk-kms", - "clap 4.5.18", + "clap 4.5.20", "const_format", "dirs 4.0.0", "dotenvy", "fuel-core", "fuel-core-chain-config", + "fuel-core-compression", "fuel-core-poa", "fuel-core-storage", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "hex", "humantime", "itertools 0.12.1", @@ -3286,7 +3307,7 @@ dependencies = [ [[package]] name = "fuel-core-chain-config" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "bech32", @@ -3294,7 +3315,7 @@ dependencies = [ "derivative", "fuel-core-chain-config", "fuel-core-storage", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "insta", "itertools 0.12.1", "parquet", @@ -3312,13 +3333,14 @@ dependencies = [ [[package]] name = "fuel-core-client" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", + "base64 0.22.1", "cynic", "derive_more", "eventsource-client", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "hex", "hyper-rustls", @@ -3335,41 +3357,58 @@ dependencies = [ [[package]] name = "fuel-core-client-bin" -version = "0.36.0" +version = "0.38.0" dependencies = [ - "clap 4.5.18", + "clap 4.5.20", "fuel-core-client", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "serde_json", "tokio", ] +[[package]] +name = "fuel-core-compression" +version = "0.38.0" +dependencies = [ + "anyhow", + "fuel-core-compression", + "fuel-core-types 0.38.0", + "paste", + "postcard", + "proptest", + "rand", + "serde", + "strum 0.25.0", + "strum_macros 0.25.3", + "tokio", +] + [[package]] name = "fuel-core-consensus-module" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "fuel-core-chain-config", "fuel-core-poa", "fuel-core-storage", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "test-case", ] [[package]] name = "fuel-core-database" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "derive_more", "fuel-core-storage", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", ] [[package]] name = "fuel-core-e2e-client" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "assert_cmd", @@ -3377,7 +3416,7 @@ dependencies = [ "fuel-core-chain-config", "fuel-core-client", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "hex", "humantime-serde", @@ -3394,12 +3433,12 @@ dependencies = [ [[package]] name = "fuel-core-executor" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "fuel-core-storage", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "hex", "parking_lot", "serde", @@ -3408,17 +3447,18 @@ dependencies = [ [[package]] name = "fuel-core-gas-price-service" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", "enum-iterator", "fuel-core-services", "fuel-core-storage", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "fuel-gas-price-algorithm", "futures", "num_enum", + "parking_lot", "reqwest", "serde", "strum 0.25.0", @@ -3431,14 +3471,14 @@ dependencies = [ [[package]] name = "fuel-core-importer" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "derive_more", "fuel-core-metrics", "fuel-core-storage", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "mockall", "parking_lot", "rayon", @@ -3449,22 +3489,22 @@ dependencies = [ [[package]] name = "fuel-core-keygen" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", - "clap 4.5.18", - "fuel-core-types 0.36.0", + "clap 4.5.20", + "fuel-core-types 0.38.0", "libp2p-identity", "serde", ] [[package]] name = "fuel-core-keygen-bin" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "atty", - "clap 4.5.18", + "clap 4.5.20", "crossterm", "fuel-core-keygen", "serde_json", @@ -3473,7 +3513,7 @@ dependencies = [ [[package]] name = "fuel-core-metrics" -version = "0.36.0" +version = "0.38.0" dependencies = [ "parking_lot", "pin-project-lite", @@ -3485,7 +3525,7 @@ dependencies = [ [[package]] name = "fuel-core-p2p" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", @@ -3496,7 +3536,7 @@ dependencies = [ "fuel-core-services", "fuel-core-storage", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "hex", "hickory-resolver", @@ -3523,7 +3563,7 @@ dependencies = [ [[package]] name = "fuel-core-poa" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", @@ -3533,7 +3573,8 @@ dependencies = [ "fuel-core-poa", "fuel-core-services", "fuel-core-storage", - "fuel-core-types 0.36.0", + "fuel-core-trace", + "fuel-core-types 0.38.0", "k256", "mockall", "rand", @@ -3547,7 +3588,7 @@ dependencies = [ [[package]] name = "fuel-core-producer" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", @@ -3555,7 +3596,7 @@ dependencies = [ "fuel-core-producer", "fuel-core-storage", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "mockall", "proptest", "rand", @@ -3566,7 +3607,7 @@ dependencies = [ [[package]] name = "fuel-core-relayer" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", @@ -3580,7 +3621,7 @@ dependencies = [ "fuel-core-services", "fuel-core-storage", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "mockall", "once_cell", @@ -3599,28 +3640,30 @@ dependencies = [ [[package]] name = "fuel-core-services" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", "fuel-core-metrics", + "fuel-core-services", "futures", "mockall", "parking_lot", + "rayon", "tokio", "tracing", ] [[package]] name = "fuel-core-storage" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "derive_more", "enum-iterator", "fuel-core-storage", - "fuel-core-types 0.36.0", - "fuel-vm 0.57.1", + "fuel-core-types 0.38.0", + "fuel-vm 0.58.2", "impl-tools", "itertools 0.12.1", "mockall", @@ -3637,13 +3680,13 @@ dependencies = [ [[package]] name = "fuel-core-sync" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", "fuel-core-services", "fuel-core-trace", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "mockall", "rand", @@ -3662,13 +3705,14 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-kms", - "clap 4.5.18", + "clap 4.5.20", "cynic", "ethers", "fuel-core", "fuel-core-benches", "fuel-core-bin", "fuel-core-client", + "fuel-core-compression", "fuel-core-executor", "fuel-core-gas-price-service", "fuel-core-p2p", @@ -3677,7 +3721,7 @@ dependencies = [ "fuel-core-storage", "fuel-core-trace", "fuel-core-txpool", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "fuel-core-upgradable-executor", "futures", "hex", @@ -3685,6 +3729,7 @@ dependencies = [ "insta", "itertools 0.12.1", "k256", + "postcard", "pretty_assertions", "primitive-types", "proptest", @@ -3702,7 +3747,7 @@ dependencies = [ [[package]] name = "fuel-core-trace" -version = "0.36.0" +version = "0.38.0" dependencies = [ "ctor", "tracing", @@ -3712,28 +3757,25 @@ dependencies = [ [[package]] name = "fuel-core-txpool" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", "derive_more", - "fuel-core-metrics", "fuel-core-services", "fuel-core-storage", "fuel-core-trace", "fuel-core-txpool", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", - "itertools 0.12.1", "mockall", "num-rational", "parking_lot", + "petgraph", "proptest", - "rayon", - "rstest", + "rand", "test-strategy", "tokio", - "tokio-rayon", "tokio-stream", "tracing", ] @@ -3756,13 +3798,13 @@ dependencies = [ [[package]] name = "fuel-core-types" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "bs58", "derivative", "derive_more", - "fuel-vm 0.57.1", + "fuel-vm 0.58.2", "rand", "secrecy", "serde", @@ -3772,13 +3814,13 @@ dependencies = [ [[package]] name = "fuel-core-upgradable-executor" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "derive_more", "fuel-core-executor", "fuel-core-storage", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "fuel-core-wasm-executor", "ntest", "parking_lot", @@ -3789,13 +3831,13 @@ dependencies = [ [[package]] name = "fuel-core-wasm-executor" -version = "0.36.0" +version = "0.38.0" dependencies = [ "anyhow", "fuel-core-executor", "fuel-core-storage", "fuel-core-types 0.35.0", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "postcard", "proptest", "serde", @@ -3820,15 +3862,15 @@ dependencies = [ [[package]] name = "fuel-crypto" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a4ca62d0a8a361b66a6eab3456285883d0aa92c407e249fc76f195c332fbb2" +checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", "ecdsa", "ed25519-dalek", - "fuel-types 0.57.1", + "fuel-types 0.58.2", "k256", "lazy_static", "p256", @@ -3853,9 +3895,9 @@ dependencies = [ [[package]] name = "fuel-derive" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca99d9c264fec231a01e55bbdb6adb3080266dc44c4cc88de870b089ee2a015" +checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" dependencies = [ "proc-macro2", "quote", @@ -3865,7 +3907,7 @@ dependencies = [ [[package]] name = "fuel-gas-price-algorithm" -version = "0.36.0" +version = "0.38.0" dependencies = [ "proptest", "rand", @@ -3890,13 +3932,13 @@ dependencies = [ [[package]] name = "fuel-merkle" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f3c08f22f7c24170a4ac255e7dfe52d52ae2e85c4a8e26ed6df4f1bd380210" +checksum = "c79eca6a452311c70978a5df796c0f99f27e474b69719e0db4c1d82e68800d07" dependencies = [ "derive_more", "digest 0.10.7", - "fuel-storage 0.57.1", + "fuel-storage 0.58.2", "hashbrown 0.13.2", "hex", "serde", @@ -3911,9 +3953,9 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d676ea5d926bf90c8ea442e5f6fc23c0e86c2e4d663f303842b4afe5b7928084" +checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" [[package]] name = "fuel-tx" @@ -3939,23 +3981,23 @@ dependencies = [ [[package]] name = "fuel-tx" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86158d08fe2a936ac765a1e2cb9bf60be704203b3085882e7cc6cd4c48de38ef" +checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" dependencies = [ "bitflags 2.6.0", "derivative", "derive_more", - "fuel-asm 0.57.1", - "fuel-crypto 0.57.1", - "fuel-merkle 0.57.1", - "fuel-types 0.57.1", + "fuel-asm 0.58.2", + "fuel-compression", + "fuel-crypto 0.58.2", + "fuel-merkle 0.58.2", + "fuel-types 0.58.2", "hashbrown 0.14.5", "itertools 0.10.5", "postcard", "rand", "serde", - "serde_json", "strum 0.24.1", "strum_macros 0.24.3", ] @@ -3973,11 +4015,11 @@ dependencies = [ [[package]] name = "fuel-types" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42b4430c6a20669fffd8f2c17d2be6809a356f623243a78b5eedf8ba7897c036" +checksum = "982265415a99b5bd6277bc24194a233bb2e18764df11c937b3dbb11a02c9e545" dependencies = [ - "fuel-derive 0.57.1", + "fuel-derive 0.58.2", "hex", "rand", "serde", @@ -4016,9 +4058,9 @@ dependencies = [ [[package]] name = "fuel-vm" -version = "0.57.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b04a4e971d58b3662ff8b84b3176094e25f4b372728a13ea903fdf79c7e426" +checksum = "54b5362d7d072c72eec20581f67fc5400090c356a7f3ae77c79880b3b177b667" dependencies = [ "anyhow", "async-trait", @@ -4027,12 +4069,13 @@ dependencies = [ "derivative", "derive_more", "ethnum", - "fuel-asm 0.57.1", - "fuel-crypto 0.57.1", - "fuel-merkle 0.57.1", - "fuel-storage 0.57.1", - "fuel-tx 0.57.1", - "fuel-types 0.57.1", + "fuel-asm 0.58.2", + "fuel-compression", + "fuel-crypto 0.58.2", + "fuel-merkle 0.58.2", + "fuel-storage 0.58.2", + "fuel-tx 0.58.2", + "fuel-types 0.58.2", "hashbrown 0.14.5", "itertools 0.10.5", "libm", @@ -4056,9 +4099,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -4081,9 +4124,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -4091,15 +4134,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -4109,9 +4152,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -4153,9 +4196,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -4169,21 +4212,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.13", + "rustls 0.23.14", "rustls-pki-types", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-ticker" @@ -4208,9 +4251,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -4272,15 +4315,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator", - "indexmap 2.5.0", + "indexmap 2.6.0", "stable_deref_trait", ] [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -4345,7 +4388,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -4397,6 +4440,17 @@ dependencies = [ "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashers" version = "1.0.1" @@ -4637,9 +4691,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -4911,12 +4965,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -5000,9 +5054,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-terminal" @@ -5065,9 +5119,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "0cb94a0ffd3f3ee755c20f7d8752f45cac88605a4dcf808abcff72873296ec7b" dependencies = [ "wasm-bindgen", ] @@ -5137,7 +5191,7 @@ dependencies = [ "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "string_cache", "term", "tiny-keccak", @@ -5151,7 +5205,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" dependencies = [ - "regex-automata 0.4.7", + "regex-automata 0.4.8", ] [[package]] @@ -5538,7 +5592,7 @@ dependencies = [ "quinn", "rand", "ring 0.17.8", - "rustls 0.23.13", + "rustls 0.23.14", "socket2 0.5.7", "thiserror", "tokio", @@ -5651,7 +5705,7 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.8", - "rustls 0.23.13", + "rustls 0.23.14", "rustls-webpki 0.101.7", "thiserror", "x509-parser", @@ -5801,7 +5855,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7" dependencies = [ - "clap 4.5.18", + "clap 4.5.20", "termcolor", "threadpool", ] @@ -5856,11 +5910,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -6395,13 +6449,13 @@ checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "crc32fast", - "hashbrown 0.14.5", - "indexmap 2.5.0", + "hashbrown 0.15.0", + "indexmap 2.6.0", "memchr", ] @@ -6416,9 +6470,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -6558,7 +6612,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.6", + "redox_syscall 0.5.7", "smallvec", "windows-targets 0.52.6", ] @@ -6676,7 +6730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.5.0", + "indexmap 2.6.0", ] [[package]] @@ -6742,18 +6796,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -7060,9 +7114,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -7104,7 +7158,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -7256,7 +7310,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustls 0.23.14", "socket2 0.5.7", "thiserror", "tokio", @@ -7273,7 +7327,7 @@ dependencies = [ "rand", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustls 0.23.14", "slab", "thiserror", "tinyvec", @@ -7349,9 +7403,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -7399,9 +7453,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] @@ -7438,14 +7492,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -7459,13 +7513,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -7482,9 +7536,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -7750,9 +7804,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "once_cell", "ring 0.17.8", @@ -7889,9 +7943,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -8097,15 +8151,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -8115,9 +8169,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling 0.20.10", "proc-macro2", @@ -8131,7 +8185,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -8560,9 +8614,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "12.11.1" +version = "12.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fdf97c441f18a4f92425b896a4ec7a27e03631a0b1047ec4e34e9916a9a167e" +checksum = "366f1b4c6baf6cfefc234bbd4899535fca0b06c74443039a73f6dfb2fad88d77" dependencies = [ "debugid", "memmap2", @@ -8572,9 +8626,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.11.1" +version = "12.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc8ece6b129e97e53d1fbb3f61d33a6a9e5369b11d01228c068094d6d134eaea" +checksum = "aba05ba5b9962ea5617baf556293720a8b2d0a282aa14ee4bf10e22efc7da8c8" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -8751,7 +8805,7 @@ name = "test-helpers" version = "0.0.0" dependencies = [ "anyhow", - "clap 4.5.18", + "clap 4.5.20", "fuel-core", "fuel-core-bin", "fuel-core-client", @@ -8761,7 +8815,7 @@ dependencies = [ "fuel-core-storage", "fuel-core-trace", "fuel-core-txpool", - "fuel-core-types 0.36.0", + "fuel-core-types 0.38.0", "futures", "itertools 0.12.1", "rand", @@ -9058,7 +9112,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -9076,6 +9130,7 @@ dependencies = [ "pin-project", "pin-project-lite", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -9095,10 +9150,28 @@ dependencies = [ "http-body 0.4.6", "http-range-header", "pin-project-lite", - "tokio", "tower", "tower-layer", "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "http-range-header", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", "tracing", ] @@ -9257,9 +9330,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "uint" @@ -9281,9 +9354,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -9491,9 +9564,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "ef073ced962d62984fb38a36e5fdc1a2b23c9e0e1fa0689bb97afa4202ef6887" dependencies = [ "cfg-if", "once_cell", @@ -9502,9 +9575,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "c4bfab14ef75323f4eb75fa52ee0a3fb59611977fd3240da19b2cf36ff85030e" dependencies = [ "bumpalo", "log", @@ -9517,9 +9590,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "65471f79c1022ffa5291d33520cbbb53b7687b01c2f8e83b57d102eed7ed479d" dependencies = [ "cfg-if", "js-sys", @@ -9529,9 +9602,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "a7bec9830f60924d9ceb3ef99d55c155be8afa76954edffbb5936ff4509474e7" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9539,9 +9612,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "4c74f6e152a76a2ad448e223b0fc0b6b5747649c3d769cc6bf45737bf97d0ed6" dependencies = [ "proc-macro2", "quote", @@ -9552,9 +9625,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "a42f6c679374623f295a8623adfe63d9284091245c3504bde47c17a3ce2777d9" [[package]] name = "wasm-encoder" @@ -9574,7 +9647,7 @@ dependencies = [ "ahash", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "semver", "serde", ] @@ -9592,9 +9665,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07232e0b473af36112da7348f51e73fa8b11047a6cb546096da3812930b7c93a" +checksum = "fe501caefeb9f7b15360bdd7e47ad96e20223846f1c7db485ae5820ba5acc3d2" dependencies = [ "anyhow", "bitflags 2.6.0", @@ -9602,7 +9675,7 @@ dependencies = [ "cc", "cfg-if", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "libc", "libm", "log", @@ -9634,18 +9707,18 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a9c42562d879c749288d9a26acc0d95d2ca069e30c2ec2efce84461c4d62b3" +checksum = "c904a057d74bfa0ad9369a3fd99231d81ba0345f059d03c9148c3bb2abbf310f" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-cache" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d5d5aac98c8ae87cf5244495da7722e3fa022aa6f3f4fcd5e3d6e5699ce422" +checksum = "8dff4d467d6b5bd0d137f5426f45178222e40b59e49ab3a7361420262b9f00df" dependencies = [ "anyhow", "base64 0.21.7", @@ -9663,9 +9736,9 @@ dependencies = [ [[package]] name = "wasmtime-component-macro" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3f57c4bc96f9b4a6ff4d6cb6e837913eff32e98d09e2b6d79b5c4647b415b" +checksum = "3a96185dab1c14ffb986ff2b3a2185d15acf2b801ca7895aa35ee80328e2ce38" dependencies = [ "anyhow", "proc-macro2", @@ -9678,15 +9751,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1da707969bc31a565da9b32d087eb2370c95c6f2087c5539a15f2e3b27e77203" +checksum = "71a40200d42a8985edadb4007a0ed320756cbe28065b83e0027e39524c1b1b22" [[package]] name = "wasmtime-cranelift" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62cb6135ec46994299be711b78b03acaa9480de3715f827d450f0c947a84977c" +checksum = "b099ef9b7808fa8d18cad32243e78e9c07a4a8aacfa913d88dc08704b1643c49" dependencies = [ "anyhow", "cfg-if", @@ -9708,15 +9781,15 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bcaa3b42a0718e9123da7fb75e8e13fc95df7db2a7e32e2f2f4f0d3333b7d6f" +checksum = "e2f1765f6ca1a166927bee13ad4aed7bf18269f34c0cd7d6d523889a0b52e6ee" dependencies = [ "anyhow", "cranelift-bitset", "cranelift-entity", "gimli 0.28.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "log", "object", "postcard", @@ -9731,9 +9804,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfee42dac5148fc2664ab1f5cb8d7fa77a28d1a2cf1d9483abc2c3d751a58b9" +checksum = "1e1a826e4ccd0803b2f7463289cad104f40d09d06bc8acf1a614230a47b4d96f" dependencies = [ "anyhow", "cfg-if", @@ -9743,15 +9816,15 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42eb8f6515708ec67974998c3e644101db4186308985f5ef7c2ef324ff33c948" +checksum = "f92a137c17c992eb5eaacfa0f0590353471e49dbb4bdbdf9cf7536d66109e63a" [[package]] name = "wasmtime-types" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046873fb8fb3e9652f3fd76fe99c8c8129007695c3d73b2e307fdae40f6e324c" +checksum = "a6072ac3267866d99ca726b6a4f157df9b733aac8082e902d527368f07c303ba" dependencies = [ "anyhow", "cranelift-entity", @@ -9763,9 +9836,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c02af2e9dbeb427304d1a08787d70ed0dbfec1af2236616f84c9f1f03e7969" +checksum = "a2bde986038b819bc43a21fef0610aeb47aabfe3ea09ca3533a7b81023b84ec6" dependencies = [ "proc-macro2", "quote", @@ -9774,21 +9847,21 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "23.0.2" +version = "23.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f528f8b8a2376a3dacaf497d960216dd466d324425361e1e00e26de0a7705c" +checksum = "8f88e49a9b81746ec0cede5505e40a4012c92cb5054cd7ef4300dc57c36f26b1" dependencies = [ "anyhow", "heck 0.4.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "wit-parser", ] [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "44188d185b5bdcae1052d08bcbcf9091a5524038d4572cc4f4f2bb9d5554ddd9" dependencies = [ "js-sys", "wasm-bindgen", @@ -10050,7 +10123,7 @@ checksum = "ceeb0424aa8679f3fcf2d6e3cfa381f3d6fa6179976a2c05a6249dd2bb426716" dependencies = [ "anyhow", "id-arena", - "indexmap 2.5.0", + "indexmap 2.6.0", "log", "semver", "serde", @@ -10142,7 +10215,7 @@ dependencies = [ name = "xtask" version = "0.0.0" dependencies = [ - "clap 4.5.18", + "clap 4.5.20", "fuel-core", ] diff --git a/Cargo.toml b/Cargo.toml index 2aaf0971300..cda1005738f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "bin/keygen", "crates/chain-config", "crates/client", + "crates/compression", "crates/database", "crates/fuel-core", "crates/fuel-gas-price-algorithm", @@ -24,7 +25,7 @@ members = [ "crates/services/producer", "crates/services/relayer", "crates/services/sync", - "crates/services/txpool", + "crates/services/txpool_v2", "crates/services/upgradable-executor", "crates/services/upgradable-executor/wasm-executor", "crates/storage", @@ -50,42 +51,43 @@ homepage = "https://fuel.network/" keywords = ["blockchain", "cryptocurrencies", "fuel-vm", "vm"] license = "BUSL-1.1" repository = "https://github.com/FuelLabs/fuel-core" -version = "0.36.0" +version = "0.38.0" [workspace.dependencies] # Workspace members -fuel-core = { version = "0.36.0", path = "./crates/fuel-core", default-features = false } -fuel-core-client-bin = { version = "0.36.0", path = "./bin/fuel-core-client" } -fuel-core-bin = { version = "0.36.0", path = "./bin/fuel-core" } -fuel-core-keygen = { version = "0.36.0", path = "./crates/keygen" } -fuel-core-keygen-bin = { version = "0.36.0", path = "./bin/keygen" } -fuel-core-chain-config = { version = "0.36.0", path = "./crates/chain-config", default-features = false } -fuel-core-client = { version = "0.36.0", path = "./crates/client" } -fuel-core-database = { version = "0.36.0", path = "./crates/database" } -fuel-core-metrics = { version = "0.36.0", path = "./crates/metrics" } -fuel-core-services = { version = "0.36.0", path = "./crates/services" } -fuel-core-consensus-module = { version = "0.36.0", path = "./crates/services/consensus_module" } -fuel-core-bft = { version = "0.36.0", path = "./crates/services/consensus_module/bft" } -fuel-core-poa = { version = "0.36.0", path = "./crates/services/consensus_module/poa" } -fuel-core-executor = { version = "0.36.0", path = "./crates/services/executor", default-features = false } -fuel-core-importer = { version = "0.36.0", path = "./crates/services/importer" } -fuel-core-gas-price-service = { version = "0.36.0", path = "crates/services/gas_price_service" } -fuel-core-p2p = { version = "0.36.0", path = "./crates/services/p2p" } -fuel-core-producer = { version = "0.36.0", path = "./crates/services/producer" } -fuel-core-relayer = { version = "0.36.0", path = "./crates/services/relayer" } -fuel-core-sync = { version = "0.36.0", path = "./crates/services/sync" } -fuel-core-txpool = { version = "0.36.0", path = "./crates/services/txpool" } -fuel-core-storage = { version = "0.36.0", path = "./crates/storage", default-features = false } -fuel-core-trace = { version = "0.36.0", path = "./crates/trace" } -fuel-core-types = { version = "0.36.0", path = "./crates/types", default-features = false } +fuel-core = { version = "0.38.0", path = "./crates/fuel-core", default-features = false } +fuel-core-client-bin = { version = "0.38.0", path = "./bin/fuel-core-client" } +fuel-core-bin = { version = "0.38.0", path = "./bin/fuel-core" } +fuel-core-keygen = { version = "0.38.0", path = "./crates/keygen" } +fuel-core-keygen-bin = { version = "0.38.0", path = "./bin/keygen" } +fuel-core-chain-config = { version = "0.38.0", path = "./crates/chain-config", default-features = false } +fuel-core-client = { version = "0.38.0", path = "./crates/client" } +fuel-core-compression = { version = "0.38.0", path = "./crates/compression" } +fuel-core-database = { version = "0.38.0", path = "./crates/database" } +fuel-core-metrics = { version = "0.38.0", path = "./crates/metrics" } +fuel-core-services = { version = "0.38.0", path = "./crates/services" } +fuel-core-consensus-module = { version = "0.38.0", path = "./crates/services/consensus_module" } +fuel-core-bft = { version = "0.38.0", path = "./crates/services/consensus_module/bft" } +fuel-core-poa = { version = "0.38.0", path = "./crates/services/consensus_module/poa" } +fuel-core-executor = { version = "0.38.0", path = "./crates/services/executor", default-features = false } +fuel-core-importer = { version = "0.38.0", path = "./crates/services/importer" } +fuel-core-gas-price-service = { version = "0.38.0", path = "crates/services/gas_price_service" } +fuel-core-p2p = { version = "0.38.0", path = "./crates/services/p2p" } +fuel-core-producer = { version = "0.38.0", path = "./crates/services/producer" } +fuel-core-relayer = { version = "0.38.0", path = "./crates/services/relayer" } +fuel-core-sync = { version = "0.38.0", path = "./crates/services/sync" } +fuel-core-txpool = { version = "0.38.0", path = "./crates/services/txpool_v2" } +fuel-core-storage = { version = "0.38.0", path = "./crates/storage", default-features = false } +fuel-core-trace = { version = "0.38.0", path = "./crates/trace" } +fuel-core-types = { version = "0.38.0", path = "./crates/types", default-features = false } fuel-core-tests = { version = "0.0.0", path = "./tests" } -fuel-core-upgradable-executor = { version = "0.36.0", path = "./crates/services/upgradable-executor" } -fuel-core-wasm-executor = { version = "0.36.0", path = "./crates/services/upgradable-executor/wasm-executor", default-features = false } +fuel-core-upgradable-executor = { version = "0.38.0", path = "./crates/services/upgradable-executor" } +fuel-core-wasm-executor = { version = "0.38.0", path = "./crates/services/upgradable-executor/wasm-executor", default-features = false } fuel-core-xtask = { version = "0.0.0", path = "./xtask" } -fuel-gas-price-algorithm = { version = "0.36.0", path = "crates/fuel-gas-price-algorithm" } +fuel-gas-price-algorithm = { version = "0.38.0", path = "crates/fuel-gas-price-algorithm" } # Fuel dependencies -fuel-vm-private = { version = "0.57.0", package = "fuel-vm", default-features = false } +fuel-vm-private = { version = "0.58.2", package = "fuel-vm", default-features = false } # Common dependencies anyhow = "1.0" @@ -112,7 +114,7 @@ postcard = "1.0" tracing-attributes = "0.1" tracing-subscriber = "0.3" serde = "1.0" -serde_json = { version = "1.0", default-features = false } +serde_json = { version = "1.0", default-features = false, features = ["alloc"] } serde_with = { version = "3.4", default-features = false } strum = { version = "0.25" } strum_macros = "0.25" @@ -129,6 +131,7 @@ test-strategy = "0.3" parquet = { version = "49.0", default-features = false } rayon = "1.10.0" bytes = "1.5.0" +paste = "1.0" pretty_assertions = "1.4.0" proptest = "1.1" pin-project-lite = "0.2" diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index 8cb4d1fd9a7..9574720a3df 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -27,7 +27,6 @@ use fuel_core::{ Config, FuelService, }, - txpool::types::Word, }; use fuel_core_benches::{ default_gas_costs::default_gas_costs, @@ -57,6 +56,7 @@ use fuel_core_types::{ GTFArgs, Instruction, RegId, + Word, }, fuel_crypto::{ secp256r1, @@ -83,6 +83,7 @@ use fuel_core_types::{ checked_transaction::EstimatePredicates, consts::WORD_SIZE, interpreter::MemoryInstance, + predicate::EmptyStorage, }, services::executor::TransactionExecutionResult, }; @@ -418,6 +419,7 @@ fn run_with_service_with_extra_inputs( tx.estimate_predicates( &chain_config.consensus_parameters.clone().into(), MemoryInstance::new(), + &EmptyStorage, ) .unwrap(); async move { @@ -426,11 +428,8 @@ fn run_with_service_with_extra_inputs( let mut sub = shared.block_importer.block_importer.subscribe(); shared .txpool_shared_state - .insert(vec![std::sync::Arc::new(tx)]) + .insert(tx) .await - .into_iter() - .next() - .expect("Should be at least 1 element") .expect("Should include transaction successfully"); let res = sub.recv().await.expect("Should produce a block"); assert_eq!(res.tx_status.len(), 2, "res.tx_status: {:?}", res.tx_status); diff --git a/benches/benches/transaction_throughput.rs b/benches/benches/transaction_throughput.rs index 55a2f11990e..23454dda359 100644 --- a/benches/benches/transaction_throughput.rs +++ b/benches/benches/transaction_throughput.rs @@ -39,6 +39,7 @@ use fuel_core_types::{ EstimatePredicates, }, interpreter::MemoryInstance, + predicate::EmptyStorage, }, }; use rand::{ @@ -103,20 +104,15 @@ where for _ in 0..iters { let mut test_builder = test_builder.clone(); let sealed_block = { - let transactions = transactions - .iter() - .map(|tx| Arc::new(tx.clone())) - .collect(); + let transactions: Vec = + transactions.iter().cloned().collect(); // start the producer node let TestContext { srv, client, .. } = test_builder.finalize().await; // insert all transactions - let results = - srv.shared.txpool_shared_state.insert(transactions).await; - for result in results { - let result = result.expect("Should insert transaction"); - assert_eq!(result.removed.len(), 0); + for tx in transactions { + srv.shared.txpool_shared_state.insert(tx).await.unwrap(); } let _ = client.produce_blocks(1, None).await; @@ -194,8 +190,12 @@ fn signed_transfers(c: &mut Criterion) { .add_output(Output::coin(rng.gen(), 50, AssetId::default())) .add_output(Output::change(rng.gen(), 0, AssetId::default())) .finalize(); - tx.estimate_predicates(&checked_parameters(), MemoryInstance::new()) - .expect("Predicate check failed"); + tx.estimate_predicates( + &checked_parameters(), + MemoryInstance::new(), + &EmptyStorage, + ) + .expect("Predicate check failed"); tx }; bench_txs("signed transfers", c, generator); @@ -220,8 +220,12 @@ fn predicate_transfers(c: &mut Criterion) { .add_output(Output::coin(rng.gen(), 50, AssetId::default())) .add_output(Output::change(rng.gen(), 0, AssetId::default())) .finalize(); - tx.estimate_predicates(&checked_parameters(), MemoryInstance::new()) - .expect("Predicate check failed"); + tx.estimate_predicates( + &checked_parameters(), + MemoryInstance::new(), + &EmptyStorage, + ) + .expect("Predicate check failed"); tx }; bench_txs("predicate transfers", c, generator); @@ -288,8 +292,12 @@ fn predicate_transfers_eck1(c: &mut Criterion) { .add_output(Output::coin(rng.gen(), 50, AssetId::default())) .add_output(Output::change(rng.gen(), 0, AssetId::default())) .finalize(); - tx.estimate_predicates(&checked_parameters(), MemoryInstance::new()) - .expect("Predicate check failed"); + tx.estimate_predicates( + &checked_parameters(), + MemoryInstance::new(), + &EmptyStorage, + ) + .expect("Predicate check failed"); tx }; bench_txs("predicate transfers eck1", c, generator); @@ -358,8 +366,12 @@ fn predicate_transfers_ed19(c: &mut Criterion) { .add_output(Output::coin(rng.gen(), 50, AssetId::default())) .add_output(Output::change(rng.gen(), 0, AssetId::default())) .finalize(); - tx.estimate_predicates(&checked_parameters(), MemoryInstance::new()) - .expect("Predicate check failed"); + tx.estimate_predicates( + &checked_parameters(), + MemoryInstance::new(), + &EmptyStorage, + ) + .expect("Predicate check failed"); tx }; bench_txs("predicate transfers ed19", c, generator); diff --git a/benches/src/default_gas_costs.rs b/benches/src/default_gas_costs.rs index 4ef866f9a27..ed10c1c3333 100644 --- a/benches/src/default_gas_costs.rs +++ b/benches/src/default_gas_costs.rs @@ -57,8 +57,8 @@ pub fn default_gas_costs() -> GasCostsValues { ori: 2, poph: 2, popl: 2, - pshh: 3073, - pshl: 3016, + pshh: 5, + pshl: 5, move_op: 2, ret: 43, sb: 2, diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 4aa68c51656..c84fb4e6418 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -494,7 +494,7 @@ impl TryFrom for VmBenchPrepared { }); // add at least one coin input - tx.add_random_fee_input(); + tx.add_fee_input(); let mut tx = tx .script_gas_limit(gas_limit) @@ -504,6 +504,7 @@ impl TryFrom for VmBenchPrepared { tx.estimate_predicates( &CheckPredicateParams::from(¶ms), MemoryInstance::new(), + db.database_mut(), ) .unwrap(); let tx = tx.into_checked(height, ¶ms).unwrap(); diff --git a/bin/e2e-test-client/tests/integration_tests.rs b/bin/e2e-test-client/tests/integration_tests.rs index 794d54d2e95..a6a13f75f2a 100644 --- a/bin/e2e-test-client/tests/integration_tests.rs +++ b/bin/e2e-test-client/tests/integration_tests.rs @@ -4,12 +4,12 @@ use fuel_core::service::{ }; // Add methods on commands -use fuel_core::txpool::types::ContractId; use fuel_core_chain_config::{ SnapshotMetadata, SnapshotReader, }; use fuel_core_e2e_client::config::SuiteConfig; +use fuel_core_types::fuel_tx::ContractId; use std::{ fs, str::FromStr, diff --git a/bin/fuel-core/Cargo.toml b/bin/fuel-core/Cargo.toml index a15848bb37e..aa111e7637c 100644 --- a/bin/fuel-core/Cargo.toml +++ b/bin/fuel-core/Cargo.toml @@ -27,6 +27,7 @@ dirs = "4.0" dotenvy = { version = "0.15", optional = true } fuel-core = { workspace = true, features = ["wasm-executor"] } fuel-core-chain-config = { workspace = true } +fuel-core-compression = { workspace = true } fuel-core-poa = { workspace = true } fuel-core-types = { workspace = true, features = ["std"] } hex = { workspace = true } diff --git a/bin/fuel-core/chainspec/local-testnet/chain_config.json b/bin/fuel-core/chainspec/local-testnet/chain_config.json index e6325f21e1b..69ac7a14b7f 100644 --- a/bin/fuel-core/chainspec/local-testnet/chain_config.json +++ b/bin/fuel-core/chainspec/local-testnet/chain_config.json @@ -93,8 +93,8 @@ "ori": 2, "poph": 2, "popl": 2, - "pshh": 3073, - "pshl": 3016, + "pshh": 5, + "pshl": 5, "ret_contract": 43, "rvrt_contract": 39, "sb": 2, @@ -297,7 +297,7 @@ "privileged_address": "9f0e19d6c2a6283a3222426ab2630d35516b1799b503f37b02105bebe1b8a3e9" } }, - "genesis_state_transition_version": 11, + "genesis_state_transition_version": 14, "consensus": { "PoAV2": { "genesis_signing_key": "e0a9fcde1b73f545252e01b30b50819eb9547d07531fa3df0385c5695736634d", diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 44be9d57a98..c88e77d1132 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -20,7 +20,10 @@ use fuel_core::{ CombinedDatabase, CombinedDatabaseConfig, }, - fuel_core_graphql_api::ServiceConfig as GraphQLConfig, + fuel_core_graphql_api::{ + worker_service::DaCompressionConfig, + ServiceConfig as GraphQLConfig, + }, producer::Config as ProducerConfig, service::{ config::Trigger, @@ -30,9 +33,12 @@ use fuel_core::{ RelayerConsensusConfig, VMConfig, }, - txpool::{ - config::BlackList, + txpool::config::{ + BlackList, Config as TxPoolConfig, + HeavyWorkConfig, + PoolLimits, + ServiceChannelLimits, }, types::{ fuel_tx::ContractId, @@ -190,6 +196,11 @@ pub struct Command { #[cfg(feature = "aws-kms")] pub consensus_aws_kms: Option, + /// If given, the node will produce and store da-compressed blocks + /// with the given retention time. + #[arg(long = "da-compression", env)] + pub da_compression: Option, + /// A new block is produced instantly when transactions are available. #[clap(flatten)] pub poa_trigger: PoATriggerArgs, @@ -272,6 +283,7 @@ impl Command { consensus_key, #[cfg(feature = "aws-kms")] consensus_aws_kms, + da_compression, poa_trigger, predefined_blocks_path, coinbase_recipient, @@ -418,30 +430,69 @@ impl Command { let block_importer = fuel_core::service::config::fuel_core_importer::Config::new(); + let da_compression = match da_compression { + Some(retention) => { + DaCompressionConfig::Enabled(fuel_core_compression::Config { + temporal_registry_retention: retention.into(), + }) + } + None => DaCompressionConfig::Disabled, + }; + let TxPoolArgs { tx_pool_ttl, + tx_ttl_check_interval, tx_max_number, - tx_max_depth, + tx_max_total_bytes, + tx_max_total_gas, + tx_max_chain_count, tx_number_active_subscriptions, tx_blacklist_addresses, tx_blacklist_coins, tx_blacklist_messages, tx_blacklist_contracts, + tx_number_threads_to_verify_transactions, + tx_size_of_verification_queue, + tx_number_threads_p2p_sync, + tx_size_of_p2p_sync_queue, + tx_max_pending_read_requests, + tx_max_pending_write_requests, } = tx_pool; - let blacklist = BlackList::new( + let black_list = BlackList::new( tx_blacklist_addresses, tx_blacklist_coins, tx_blacklist_messages, tx_blacklist_contracts, ); + let pool_limits = PoolLimits { + max_txs: tx_max_number, + max_gas: tx_max_total_gas, + max_bytes_size: tx_max_total_bytes, + }; + + let pool_heavy_work_config = HeavyWorkConfig { + number_threads_to_verify_transactions: + tx_number_threads_to_verify_transactions, + size_of_verification_queue: tx_size_of_verification_queue, + number_threads_p2p_sync: tx_number_threads_p2p_sync, + size_of_p2p_sync_queue: tx_size_of_p2p_sync_queue, + }; + + let service_channel_limits = ServiceChannelLimits { + max_pending_read_pool_requests: tx_max_pending_read_requests, + max_pending_write_pool_requests: tx_max_pending_write_requests, + }; + let config = Config { graphql_config: GraphQLConfig { addr, max_queries_depth: graphql.graphql_max_depth, max_queries_complexity: graphql.graphql_max_complexity, max_queries_recursive_depth: graphql.graphql_max_recursive_depth, + max_queries_directives: graphql.max_queries_directives, + max_concurrent_queries: graphql.graphql_max_concurrent_queries, request_body_bytes_limit: graphql.graphql_request_body_bytes_limit, api_request_timeout: graphql.api_request_timeout.into(), query_log_threshold_time: graphql.query_log_threshold_time.into(), @@ -457,15 +508,17 @@ impl Command { vm: VMConfig { backtrace: vm_backtrace, }, - txpool: TxPoolConfig::new( - tx_max_number, - tx_max_depth, + txpool: TxPoolConfig { + max_txs_chain_count: tx_max_chain_count, + max_txs_ttl: tx_pool_ttl.into(), + ttl_check_interval: tx_ttl_check_interval.into(), utxo_validation, - metrics, - tx_pool_ttl.into(), - tx_number_active_subscriptions, - blacklist, - ), + max_tx_update_subscriptions: tx_number_active_subscriptions, + black_list, + pool_limits, + heavy_work: pool_heavy_work_config, + service_channel_limits, + }, block_producer: ProducerConfig { coinbase_recipient, metrics, @@ -475,6 +528,7 @@ impl Command { min_gas_price, gas_price_threshold_percent, block_importer, + da_compression, #[cfg(feature = "relayer")] relayer: relayer_cfg, #[cfg(feature = "p2p")] diff --git a/bin/fuel-core/src/cli/run/graphql.rs b/bin/fuel-core/src/cli/run/graphql.rs index 66ab6f06886..fbc84e13cac 100644 --- a/bin/fuel-core/src/cli/run/graphql.rs +++ b/bin/fuel-core/src/cli/run/graphql.rs @@ -17,13 +17,21 @@ pub struct GraphQLArgs { pub graphql_max_depth: usize, /// The max complexity of GraphQL queries. - #[clap(long = "graphql-max-complexity", default_value = "20000", env)] + #[clap(long = "graphql-max-complexity", default_value = "80000", env)] pub graphql_max_complexity: usize, /// The max recursive depth of GraphQL queries. #[clap(long = "graphql-max-recursive-depth", default_value = "16", env)] pub graphql_max_recursive_depth: usize, + /// The max number of directives in the query. + #[clap(long = "graphql-max-directives", default_value = "10", env)] + pub max_queries_directives: usize, + + /// The max number of concurrent queries. + #[clap(long = "graphql-max-concurrent-queries", default_value = "1024", env)] + pub graphql_max_concurrent_queries: usize, + /// The max body limit of the GraphQL query. #[clap( long = "graphql-request-body-bytes-limit", diff --git a/bin/fuel-core/src/cli/run/p2p.rs b/bin/fuel-core/src/cli/run/p2p.rs index 1e91f80589a..98a9a33759c 100644 --- a/bin/fuel-core/src/cli/run/p2p.rs +++ b/bin/fuel-core/src/cli/run/p2p.rs @@ -189,6 +189,14 @@ pub struct P2PArgs { /// For peer reputations, the maximum time since last heartbeat before penalty #[clap(long = "heartbeat-max-time-since-last", default_value = "40", env)] pub heartbeat_max_time_since_last: u64, + + /// Number of threads to read from the database. + #[clap(long = "p2p-database-read-threads", default_value = "2", env)] + pub database_read_threads: usize, + + /// Number of threads to read from the tx pool. + #[clap(long = "p2p-txpool-threads", default_value = "0", env)] + pub tx_pool_threads: usize, } #[derive(Debug, Clone, Args)] @@ -335,6 +343,8 @@ impl P2PArgs { info_interval: Some(Duration::from_secs(self.info_interval)), identify_interval: Some(Duration::from_secs(self.identify_interval)), metrics, + database_read_threads: self.database_read_threads, + tx_pool_threads: self.tx_pool_threads, state: NotInitialized, }; Ok(Some(config)) diff --git a/bin/fuel-core/src/cli/run/relayer.rs b/bin/fuel-core/src/cli/run/relayer.rs index e1dd7a4ecbb..8494e7cfd7a 100644 --- a/bin/fuel-core/src/cli/run/relayer.rs +++ b/bin/fuel-core/src/cli/run/relayer.rs @@ -19,7 +19,7 @@ pub struct RelayerArgs { /// Uri addresses to ethereum client. It can be in format of `http://localhost:8545/` or `ws://localhost:8545/`. /// If not set relayer will not start. - #[arg(long = "relayer", env)] + #[arg(long = "relayer", value_delimiter = ',', env)] #[arg(required_if_eq("enable_relayer", "true"))] #[arg(requires_if(IsPresent, "enable_relayer"))] pub relayer: Option>, @@ -70,3 +70,27 @@ impl RelayerArgs { Some(config) } } + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + use test_case::test_case; + + #[derive(Debug, Clone, Parser)] + pub struct Command { + #[clap(flatten)] + relayer: RelayerArgs, + } + + #[test_case(&[""] => Ok(None); "no args")] + #[test_case(&["", "--enable-relayer", "--relayer=https://test.com"] => Ok(Some(vec![url::Url::parse("https://test.com").unwrap()])); "one relayer")] + #[test_case(&["", "--enable-relayer", "--relayer=https://test.com", "--relayer=https://test2.com"] => Ok(Some(vec![url::Url::parse("https://test.com").unwrap(), url::Url::parse("https://test2.com").unwrap()])); "two relayers in different args")] + #[test_case(&["", "--enable-relayer", "--relayer=https://test.com,https://test2.com"] => Ok(Some(vec![url::Url::parse("https://test.com").unwrap(), url::Url::parse("https://test2.com").unwrap()])); "two relayers in same arg")] + fn parse_relayer_urls(args: &[&str]) -> Result>, String> { + let command: Command = + Command::try_parse_from(args).map_err(|e| e.to_string())?; + let args = command.relayer; + Ok(args.relayer) + } +} diff --git a/bin/fuel-core/src/cli/run/tx_pool.rs b/bin/fuel-core/src/cli/run/tx_pool.rs index 844b10c333a..34f33df44c1 100644 --- a/bin/fuel-core/src/cli/run/tx_pool.rs +++ b/bin/fuel-core/src/cli/run/tx_pool.rs @@ -1,9 +1,9 @@ //! Clap configuration related to TxPool service. -use fuel_core::txpool::types::ContractId; use fuel_core_types::{ fuel_tx::{ Address, + ContractId, UtxoId, }, fuel_types::Nonce, @@ -15,13 +15,25 @@ pub struct TxPoolArgs { #[clap(long = "tx-pool-ttl", default_value = "5m", env)] pub tx_pool_ttl: humantime::Duration, + /// The interval for checking the time to live of transactions. + #[clap(long = "tx-ttl-check-interval", default_value = "1m", env)] + pub tx_ttl_check_interval: humantime::Duration, + /// The max number of transactions that the `TxPool` can simultaneously store. #[clap(long = "tx-max-number", default_value = "4064", env)] pub tx_max_number: usize, - /// The max depth of the dependent transactions that supported by the `TxPool`. - #[clap(long = "tx-max-depth", default_value = "10", env)] - pub tx_max_depth: usize, + /// The max number of gas the `TxPool` can simultaneously store. + #[clap(long = "tx-max-total-gas", default_value = "30000000000", env)] + pub tx_max_total_gas: u64, + + /// The max number of bytes that the `TxPool` can simultaneously store. + #[clap(long = "tx-max-total-bytes", default_value = "131072000", env)] + pub tx_max_total_bytes: usize, + + /// The max number of tx in a chain of dependent transactions that supported by the `TxPool`. + #[clap(long = "tx-max-depth", default_value = "32", env)] + pub tx_max_chain_count: usize, /// The maximum number of active subscriptions that supported by the `TxPool`. #[clap(long = "tx-number-active-subscriptions", default_value = "4064", env)] @@ -42,6 +54,34 @@ pub struct TxPoolArgs { /// The list of banned contracts ignored by the `TxPool`. #[clap(long = "tx-blacklist-contracts", value_delimiter = ',', env)] pub tx_blacklist_contracts: Vec, + + /// Number of threads for managing verifications/insertions. + #[clap( + long = "tx-number-threads-to-verify-transactions", + default_value = "4", + env + )] + pub tx_number_threads_to_verify_transactions: usize, + + /// Maximum number of tasks in the verifications/insertions queue. + #[clap(long = "tx-size-of-verification-queue", default_value = "2000", env)] + pub tx_size_of_verification_queue: usize, + + /// Number of threads for managing the p2p synchronisation. + #[clap(long = "tx-number-threads-p2p-sync", default_value = "2", env)] + pub tx_number_threads_p2p_sync: usize, + + /// Maximum number of tasks in the p2p synchronisation queue. + #[clap(long = "tx-size-of-p2p-sync-queue", default_value = "20", env)] + pub tx_size_of_p2p_sync_queue: usize, + + /// Maximum number of pending write requests in the service. + #[clap(long = "tx-max-pending-write-requests", default_value = "500", env)] + pub tx_max_pending_write_requests: usize, + + /// Maximum number of pending read requests in the service. + #[clap(long = "tx-max-pending-read-requests", default_value = "1000", env)] + pub tx_max_pending_read_requests: usize, } #[cfg(test)] diff --git a/crates/chain-config/src/config/snapshots/fuel_core_chain_config__config__chain__tests__snapshot_local_testnet_config.snap b/crates/chain-config/src/config/snapshots/fuel_core_chain_config__config__chain__tests__snapshot_local_testnet_config.snap index 84cb1358876..d4161b84cfd 100644 --- a/crates/chain-config/src/config/snapshots/fuel_core_chain_config__config__chain__tests__snapshot_local_testnet_config.snap +++ b/crates/chain-config/src/config/snapshots/fuel_core_chain_config__config__chain__tests__snapshot_local_testnet_config.snap @@ -301,7 +301,7 @@ expression: json "privileged_address": "0000000000000000000000000000000000000000000000000000000000000000" } }, - "genesis_state_transition_version": 11, + "genesis_state_transition_version": 14, "consensus": { "PoAV2": { "genesis_signing_key": "22ec92c3105c942a6640bdc4e4907286ec4728e8cfc0d8ac59aad4d8e1ccaefb", diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index 52a45dd62c3..ad8f7d48078 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -12,9 +12,10 @@ description = "Tx client and schema specification." [dependencies] anyhow = { workspace = true } +base64 = { version = "0.22.1", optional = true } cynic = { workspace = true } derive_more = { workspace = true } -eventsource-client = { version = "0.12.2", optional = true } +eventsource-client = { version = "0.13.0", optional = true } fuel-core-types = { workspace = true, features = ["alloc", "serde"] } futures = { workspace = true, optional = true } hex = { workspace = true } @@ -46,4 +47,4 @@ serde_json = { version = "1.0", features = ["raw_value"] } std = ["fuel-core-types/std"] default = ["subscriptions", "std"] test-helpers = [] -subscriptions = ["eventsource-client", "futures", "hyper-rustls"] +subscriptions = ["base64", "eventsource-client", "futures", "hyper-rustls"] diff --git a/crates/client/assets/schema.sdl b/crates/client/assets/schema.sdl index c7692a11adb..67287341c38 100644 --- a/crates/client/assets/schema.sdl +++ b/crates/client/assets/schema.sdl @@ -284,6 +284,10 @@ enum ContractParametersVersion { V1 } +type DaCompressedBlock { + bytes: HexString! +} + union DependentCost = LightOperation | HeavyOperation type DryRunFailureStatus { @@ -942,6 +946,12 @@ type Query { """ excludedIds: ExcludeInput ): [[CoinType!]!]! + daCompressedBlock( + """ + Height of the block + """ + height: U32! + ): DaCompressedBlock contract( """ ID of the Contract diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index 512bac9e5ce..763ceb0ac75 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -29,6 +29,11 @@ use crate::client::{ }; use anyhow::Context; #[cfg(feature = "subscriptions")] +use base64::prelude::{ + Engine as _, + BASE64_STANDARD, +}; +#[cfg(feature = "subscriptions")] use cynic::StreamingOperation; use cynic::{ http::ReqwestExt, @@ -76,6 +81,7 @@ use schema::{ block::BlockByIdArgs, coins::CoinByIdArgs, contract::ContractByIdArgs, + da_compressed::DaCompressedBlockByHeightArgs, tx::{ TxArg, TxIdArgs, @@ -270,6 +276,19 @@ impl FuelClient { format!("Failed to add header to client {e:?}"), ) })?; + if let Some(password) = url.password() { + let username = url.username(); + let credentials = format!("{}:{}", username, password); + let authorization = format!("Basic {}", BASE64_STANDARD.encode(credentials)); + client_builder = client_builder + .header("Authorization", &authorization) + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to add header to client {e:?}"), + ) + })?; + } if let Some(value) = self.cookie.deref().cookies(&self.url) { let value = value.to_str().map_err(|e| { @@ -862,6 +881,23 @@ impl FuelClient { Ok(block) } + pub async fn da_compressed_block( + &self, + height: BlockHeight, + ) -> io::Result>> { + let query = schema::da_compressed::DaCompressedBlockByHeightQuery::build( + DaCompressedBlockByHeightArgs { + height: U32(height.into()), + }, + ); + + Ok(self + .query(query) + .await? + .da_compressed_block + .map(|b| b.bytes.into())) + } + /// Retrieve a blob by its ID pub async fn blob(&self, id: BlobId) -> io::Result> { let query = schema::blob::BlobByIdQuery::build(BlobByIdArgs { id: id.into() }); @@ -869,6 +905,12 @@ impl FuelClient { Ok(blob) } + /// Check whether a blob with ID exists + pub async fn blob_exists(&self, id: BlobId) -> io::Result { + let query = schema::blob::BlobExistsQuery::build(BlobByIdArgs { id: id.into() }); + Ok(self.query(query).await?.blob.is_some()) + } + /// Retrieve multiple blocks pub async fn blocks( &self, diff --git a/crates/client/src/client/schema.rs b/crates/client/src/client/schema.rs index 648304425aa..7930d66c1ab 100644 --- a/crates/client/src/client/schema.rs +++ b/crates/client/src/client/schema.rs @@ -32,6 +32,7 @@ pub mod block; pub mod chain; pub mod coins; pub mod contract; +pub mod da_compressed; pub mod message; pub mod node_info; pub mod upgrades; diff --git a/crates/client/src/client/schema/blob.rs b/crates/client/src/client/schema/blob.rs index 3ea99f287dc..3a3b9e65b76 100644 --- a/crates/client/src/client/schema/blob.rs +++ b/crates/client/src/client/schema/blob.rs @@ -26,3 +26,20 @@ pub struct Blob { pub id: BlobId, pub bytecode: HexString, } + +#[derive(cynic::QueryFragment, Clone, Debug)] +#[cynic(schema_path = "./assets/schema.sdl", graphql_type = "Blob")] +pub struct BlobIdFragment { + pub id: BlobId, +} + +#[derive(cynic::QueryFragment, Clone, Debug)] +#[cynic( + schema_path = "./assets/schema.sdl", + graphql_type = "Query", + variables = "BlobByIdArgs" +)] +pub struct BlobExistsQuery { + #[arguments(id: $id)] + pub blob: Option, +} diff --git a/crates/client/src/client/schema/da_compressed.rs b/crates/client/src/client/schema/da_compressed.rs new file mode 100644 index 00000000000..73a131f1d32 --- /dev/null +++ b/crates/client/src/client/schema/da_compressed.rs @@ -0,0 +1,44 @@ +use crate::client::schema::{ + schema, + U32, +}; + +use super::HexString; + +#[derive(cynic::QueryVariables, Debug)] +pub struct DaCompressedBlockByHeightArgs { + pub height: U32, +} + +#[derive(cynic::QueryFragment, Clone, Debug)] +#[cynic( + schema_path = "./assets/schema.sdl", + graphql_type = "Query", + variables = "DaCompressedBlockByHeightArgs" +)] +pub struct DaCompressedBlockByHeightQuery { + #[arguments(height: $height)] + pub da_compressed_block: Option, +} + +/// Block with transaction ids +#[derive(cynic::QueryFragment, Clone, Debug)] +#[cynic(schema_path = "./assets/schema.sdl")] +pub struct DaCompressedBlock { + pub bytes: HexString, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_by_height_query_gql_output() { + use cynic::QueryBuilder; + let operation = + DaCompressedBlockByHeightQuery::build(DaCompressedBlockByHeightArgs { + height: U32(0), + }); + insta::assert_snapshot!(operation.query) + } +} diff --git a/crates/client/src/client/schema/snapshots/fuel_core_client__client__schema__da_compressed__tests__block_by_height_query_gql_output.snap b/crates/client/src/client/schema/snapshots/fuel_core_client__client__schema__da_compressed__tests__block_by_height_query_gql_output.snap new file mode 100644 index 00000000000..62f797d34e8 --- /dev/null +++ b/crates/client/src/client/schema/snapshots/fuel_core_client__client__schema__da_compressed__tests__block_by_height_query_gql_output.snap @@ -0,0 +1,9 @@ +--- +source: crates/client/src/client/schema/da_compressed.rs +expression: operation.query +--- +query($height: U32!) { + daCompressedBlock(height: $height) { + bytes + } +} diff --git a/crates/compression/Cargo.toml b/crates/compression/Cargo.toml new file mode 100644 index 00000000000..40e092d1e00 --- /dev/null +++ b/crates/compression/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "fuel-core-compression" +version = { workspace = true } +authors = { workspace = true } +categories = ["cryptography::cryptocurrencies"] +edition = { workspace = true } +homepage = { workspace = true } +keywords = [ + "blockchain", + "cryptocurrencies", + "fuel-core", + "fuel-client", + "fuel-compression", +] +license = { workspace = true } +repository = { workspace = true } +description = "Compression and decompression of Fuel blocks for DA storage." + +[dependencies] +anyhow = { workspace = true } +fuel-core-types = { workspace = true, features = [ + "alloc", + "serde", + "da-compression", +] } +paste = { workspace = true } +rand = { workspace = true, optional = true } +serde = { version = "1.0", features = ["derive"] } +strum = { workspace = true } +strum_macros = { workspace = true } + +[dev-dependencies] +fuel-core-compression = { path = ".", features = ["test-helpers"] } +postcard = { version = "1.0", features = ["use-std"] } +proptest = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[features] +test-helpers = [ + "dep:rand", + "fuel-core-types/test-helpers", + "fuel-core-types/random", + "fuel-core-types/std", +] diff --git a/crates/compression/README.md b/crates/compression/README.md new file mode 100644 index 00000000000..0768e6a1287 --- /dev/null +++ b/crates/compression/README.md @@ -0,0 +1,27 @@ +# Compression and decompression of transactions for the DA layer + +## Compressed block header + +Each compressed block begins with a version field, so that it's possible to change the format later. + +## Temporal registry + +This crate provides offchain registries for different types such as `AssetId`, `ContractId`, scripts, and predicates. Each registry is a key-value store with three-byte key. The registries are essentially compression caches. The three byte key allows cache size of 16 million values before reregistering the older values. + +The registries allow replacing repeated objects with their respective keys, so if an object +is used multiple times in a short interval (couple of months, maybe), then the full value +exists on only a single uncompressed block. + +### Fraud proofs + +Compressed block will contain a merkle root over all compression smts, followed by newly registered values along with their keys. Using an SMT provides flexibility around the algorithm we use to define keys without knowing how exactly values were chosen to be registered. + +Each registry also uses an SMT. Since the keys are three bytes long, the depth of the SMT is capped at 24 levels. + +## Compression of `UtxoIds` + +Since each `UtxoId` only appears once, there's no point in registering them. Instead, they are replaced with `TxPointer` and output index, which are still unique. + +### Fraud proofs + +During fraud proofs we need to use the `prev_root` to prove that the referenced block height is part of the chain. diff --git a/crates/compression/src/compress.rs b/crates/compression/src/compress.rs new file mode 100644 index 00000000000..0ad14e39f55 --- /dev/null +++ b/crates/compression/src/compress.rs @@ -0,0 +1,247 @@ +use crate::{ + config::Config, + eviction_policy::CacheEvictor, + ports::{ + EvictorDb, + TemporalRegistry, + UtxoIdToPointer, + }, + registry::{ + EvictorDbAll, + PerRegistryKeyspace, + RegistrationsPerTable, + TemporalRegistryAll, + }, + CompressedBlockPayloadV0, + VersionedCompressedBlock, +}; +use anyhow::Context; +use fuel_core_types::{ + blockchain::block::Block, + fuel_compression::{ + CompressibleBy, + ContextError, + RegistryKey, + }, + fuel_tx::{ + input::PredicateCode, + CompressedUtxoId, + ScriptCode, + TxPointer, + UtxoId, + }, + fuel_types::{ + Address, + AssetId, + ContractId, + }, + tai64::Tai64, +}; +use std::collections::{ + HashMap, + HashSet, +}; + +pub trait CompressDb: TemporalRegistryAll + EvictorDbAll + UtxoIdToPointer {} +impl CompressDb for T where T: TemporalRegistryAll + EvictorDbAll + UtxoIdToPointer {} + +/// This must be called for all new blocks in sequence, otherwise the result will be garbage, since +/// the registry is valid for only the current block height. On any other height you could be +/// referring to keys that have already been overwritten, or have not been written to yet. +pub async fn compress( + config: Config, + mut db: D, + block: &Block, +) -> anyhow::Result +where + D: CompressDb, +{ + let target = block.transactions_vec(); + + let mut prepare_ctx = PrepareCtx { + config, + timestamp: block.header().time(), + db: &mut db, + accessed_keys: Default::default(), + }; + let _ = target.compress_with(&mut prepare_ctx).await?; + + let mut ctx = prepare_ctx.into_compression_context()?; + let transactions = target.compress_with(&mut ctx).await?; + let registrations: RegistrationsPerTable = ctx.finalize()?; + + Ok(VersionedCompressedBlock::V0(CompressedBlockPayloadV0 { + registrations, + header: block.header().into(), + transactions, + })) +} + +/// Preparation pass through the block to collect all keys accessed during compression. +/// Returns dummy values. The resulting "compressed block" should be discarded. +struct PrepareCtx { + config: Config, + /// Current timestamp + timestamp: Tai64, + /// Database handle + db: D, + /// Keys accessed during the compression. + accessed_keys: PerRegistryKeyspace>, +} + +impl ContextError for PrepareCtx { + type Error = anyhow::Error; +} + +impl CompressibleBy> for UtxoId +where + D: CompressDb, +{ + async fn compress_with( + &self, + _ctx: &mut PrepareCtx, + ) -> anyhow::Result { + Ok(CompressedUtxoId { + tx_pointer: TxPointer::default(), + output_index: 0, + }) + } +} + +#[derive(Debug)] +struct CompressCtxKeyspace { + /// Cache evictor state for this keyspace + cache_evictor: CacheEvictor, + /// Changes to the temporary registry, to be included in the compressed block header + changes: HashMap, + /// Reverse lookup into changes + changes_lookup: HashMap, +} + +macro_rules! compression { + ($($ident:ty: $type:ty),*) => { paste::paste! { + pub struct CompressCtx { + config: Config, + timestamp: Tai64, + db: D, + $($ident: CompressCtxKeyspace<$type>,)* + } + + impl PrepareCtx where D: CompressDb { + /// Converts the preparation context into a [`CompressCtx`] + /// keeping accessed keys to avoid its eviction during compression. + /// Initializes the cache evictors from the database, which may fail. + pub fn into_compression_context(mut self) -> anyhow::Result> { + Ok(CompressCtx { + $( + $ident: CompressCtxKeyspace { + changes: Default::default(), + changes_lookup: Default::default(), + cache_evictor: CacheEvictor::new_from_db(&mut self.db, self.accessed_keys.$ident.into())?, + }, + )* + config: self.config, + timestamp: self.timestamp, + db: self.db, + }) + } + } + + impl CompressCtx where D: CompressDb { + /// Finalizes the compression context, returning the changes to the registry. + /// Commits the registrations and cache evictor states to the database. + fn finalize(mut self) -> anyhow::Result { + let mut registrations = RegistrationsPerTable::default(); + $( + self.$ident.cache_evictor.commit(&mut self.db)?; + for (key, value) in self.$ident.changes.into_iter() { + registrations.$ident.push((key, value)); + } + )* + registrations.write_to_registry(&mut self.db, self.timestamp)?; + Ok(registrations) + } + } + + $( + impl CompressibleBy> for $type + where + D: TemporalRegistry<$type> + EvictorDb<$type> + { + async fn compress_with( + &self, + ctx: &mut PrepareCtx, + ) -> anyhow::Result { + if *self == <$type>::default() { + return Ok(RegistryKey::ZERO); + } + if let Some(found) = ctx.db.registry_index_lookup(self)? { + if !ctx.accessed_keys.$ident.contains(&found) { + let key_timestamp = ctx.db.read_timestamp(&found) + .context("Database invariant violated: no timestamp stored but key found")?; + if ctx.config.is_timestamp_accessible(ctx.timestamp, key_timestamp)? { + ctx.accessed_keys.$ident.insert(found); + } + } + } + Ok(RegistryKey::ZERO) + } + } + + impl CompressibleBy> for $type + where + D: TemporalRegistry<$type> + EvictorDb<$type> + { + async fn compress_with( + &self, + ctx: &mut CompressCtx, + ) -> anyhow::Result { + if self == &Default::default() { + return Ok(RegistryKey::DEFAULT_VALUE); + } + if let Some(found) = ctx.$ident.changes_lookup.get(self) { + return Ok(*found); + } + if let Some(found) = ctx.db.registry_index_lookup(self)? { + let key_timestamp = ctx.db.read_timestamp(&found) + .context("Database invariant violated: no timestamp stored but key found")?; + if ctx.config.is_timestamp_accessible(ctx.timestamp, key_timestamp)? { + return Ok(found); + } + } + + let key = ctx.$ident.cache_evictor.next_key(); + let old = ctx.$ident.changes.insert(key, self.clone()); + let old_rev = ctx.$ident.changes_lookup.insert(self.clone(), key); + debug_assert!(old.is_none(), "Key collision in registry substitution"); + debug_assert!(old_rev.is_none(), "Key collision in registry substitution"); + Ok(key) + } + } + )* + }}; +} + +compression!( + address: Address, + asset_id: AssetId, + contract_id: ContractId, + script_code: ScriptCode, + predicate_code: PredicateCode +); + +impl ContextError for CompressCtx { + type Error = anyhow::Error; +} + +impl CompressibleBy> for UtxoId +where + D: CompressDb, +{ + async fn compress_with( + &self, + ctx: &mut CompressCtx, + ) -> anyhow::Result { + ctx.db.lookup(*self) + } +} diff --git a/crates/compression/src/config.rs b/crates/compression/src/config.rs new file mode 100644 index 00000000000..cc111119c59 --- /dev/null +++ b/crates/compression/src/config.rs @@ -0,0 +1,33 @@ +use core::time::Duration; + +use fuel_core_types::tai64::{ + Tai64, + Tai64N, +}; + +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// How long entries in the temporal registry are valid. + /// After this time has passed, the entry is considered stale and must not be used. + /// If the value is needed again, it must be re-registered. + pub temporal_registry_retention: Duration, +} + +impl Config { + /// Given timestamp of the current block and a key in an older block, + /// is the key is still accessible? + /// Returns error if the arguments are not valid block timestamps, + /// or if the block is older than the key. + pub fn is_timestamp_accessible( + &self, + block_timestamp: Tai64, + key_timestamp: Tai64, + ) -> anyhow::Result { + let block = Tai64N(block_timestamp, 0); + let key = Tai64N(key_timestamp, 0); + let duration = block + .duration_since(&key) + .map_err(|_| anyhow::anyhow!("Invalid timestamp ordering"))?; + Ok(duration <= self.temporal_registry_retention) + } +} diff --git a/crates/compression/src/decompress.rs b/crates/compression/src/decompress.rs new file mode 100644 index 00000000000..15565ce8433 --- /dev/null +++ b/crates/compression/src/decompress.rs @@ -0,0 +1,336 @@ +use crate::{ + config::Config, + ports::{ + HistoryLookup, + TemporalRegistry, + }, + registry::TemporalRegistryAll, + VersionedCompressedBlock, +}; +use fuel_core_types::{ + blockchain::block::PartialFuelBlock, + fuel_compression::{ + Compressible, + ContextError, + Decompress, + DecompressibleBy, + RegistryKey, + }, + fuel_tx::{ + input::{ + coin::{ + Coin, + CoinSpecification, + }, + message::{ + Message, + MessageSpecification, + }, + AsField, + PredicateCode, + }, + AssetId, + CompressedUtxoId, + Mint, + ScriptCode, + Transaction, + UtxoId, + }, + fuel_types::{ + Address, + ContractId, + }, + tai64::Tai64, +}; + +pub trait DecompressDb: TemporalRegistryAll + HistoryLookup {} +impl DecompressDb for T where T: TemporalRegistryAll + HistoryLookup {} + +/// This must be called for all decompressed blocks in sequence, otherwise the result will be garbage. +pub async fn decompress( + config: Config, + mut db: D, + block: VersionedCompressedBlock, +) -> anyhow::Result +where + D: DecompressDb, +{ + let VersionedCompressedBlock::V0(compressed) = block; + + // TODO: merkle root verification: https://github.com/FuelLabs/fuel-core/issues/2232 + + compressed + .registrations + .write_to_registry(&mut db, compressed.header.consensus.time)?; + + let ctx = DecompressCtx { + config, + timestamp: compressed.header.consensus.time, + db, + }; + + let transactions = as DecompressibleBy<_>>::decompress_with( + compressed.transactions, + &ctx, + ) + .await?; + + Ok(PartialFuelBlock { + header: compressed.header, + transactions, + }) +} + +pub struct DecompressCtx { + pub config: Config, + /// Timestamp of the block being decompressed + pub timestamp: Tai64, + pub db: D, +} + +impl ContextError for DecompressCtx { + type Error = anyhow::Error; +} + +impl DecompressibleBy> for UtxoId +where + D: HistoryLookup, +{ + async fn decompress_with( + c: CompressedUtxoId, + ctx: &DecompressCtx, + ) -> anyhow::Result { + ctx.db.utxo_id(c) + } +} + +macro_rules! decompress_impl { + ($($type:ty),*) => { paste::paste! { + $( + impl DecompressibleBy> for $type + where + D: TemporalRegistry<$type> + { + async fn decompress_with( + key: RegistryKey, + ctx: &DecompressCtx, + ) -> anyhow::Result { + if key == RegistryKey::DEFAULT_VALUE { + return Ok(<$type>::default()); + } + let key_timestamp = ctx.db.read_timestamp(&key)?; + if !ctx.config.is_timestamp_accessible(ctx.timestamp, key_timestamp)? { + anyhow::bail!("Timestamp not accessible"); + } + ctx.db.read_registry(&key) + } + } + )* + }}; +} + +decompress_impl!(AssetId, ContractId, Address, PredicateCode, ScriptCode); + +impl DecompressibleBy> for Coin +where + D: DecompressDb, + Specification: CoinSpecification, + Specification::Predicate: DecompressibleBy>, + Specification::PredicateData: DecompressibleBy>, + Specification::PredicateGasUsed: DecompressibleBy>, + Specification::Witness: DecompressibleBy>, +{ + async fn decompress_with( + c: as Compressible>::Compressed, + ctx: &DecompressCtx, + ) -> anyhow::Result> { + let utxo_id = UtxoId::decompress_with(c.utxo_id, ctx).await?; + let coin_info = ctx.db.coin(utxo_id)?; + let witness_index = c.witness_index.decompress(ctx).await?; + let predicate_gas_used = c.predicate_gas_used.decompress(ctx).await?; + let predicate = c.predicate.decompress(ctx).await?; + let predicate_data = c.predicate_data.decompress(ctx).await?; + Ok(Self { + utxo_id, + owner: coin_info.owner, + amount: coin_info.amount, + asset_id: coin_info.asset_id, + tx_pointer: Default::default(), + witness_index, + predicate_gas_used, + predicate, + predicate_data, + }) + } +} + +impl DecompressibleBy> for Message +where + D: DecompressDb, + Specification: MessageSpecification, + Specification::Data: DecompressibleBy> + Default, + Specification::Predicate: DecompressibleBy>, + Specification::PredicateData: DecompressibleBy>, + Specification::PredicateGasUsed: DecompressibleBy>, + Specification::Witness: DecompressibleBy>, +{ + async fn decompress_with( + c: as Compressible>::Compressed, + ctx: &DecompressCtx, + ) -> anyhow::Result> { + let msg = ctx.db.message(c.nonce)?; + let witness_index = c.witness_index.decompress(ctx).await?; + let predicate_gas_used = c.predicate_gas_used.decompress(ctx).await?; + let predicate = c.predicate.decompress(ctx).await?; + let predicate_data = c.predicate_data.decompress(ctx).await?; + let mut message: Message = Message { + sender: msg.sender, + recipient: msg.recipient, + amount: msg.amount, + nonce: c.nonce, + witness_index, + predicate_gas_used, + data: Default::default(), + predicate, + predicate_data, + }; + + if let Some(data) = message.data.as_mut_field() { + data.clone_from(&msg.data) + } + + Ok(message) + } +} + +impl DecompressibleBy> for Mint +where + D: DecompressDb, +{ + async fn decompress_with( + c: Self::Compressed, + ctx: &DecompressCtx, + ) -> anyhow::Result { + Ok(Transaction::mint( + Default::default(), // TODO: what should this we do with this? + c.input_contract.decompress(ctx).await?, + c.output_contract.decompress(ctx).await?, + c.mint_amount.decompress(ctx).await?, + c.mint_asset_id.decompress(ctx).await?, + c.gas_price.decompress(ctx).await?, + )) + } +} + +#[cfg(test)] +mod tests { + use crate::ports::{ + EvictorDb, + TemporalRegistry, + }; + + use super::*; + use fuel_core_types::{ + fuel_compression::RegistryKey, + fuel_tx::{ + input::PredicateCode, + Address, + AssetId, + ContractId, + ScriptCode, + }, + }; + use serde::{ + Deserialize, + Serialize, + }; + + pub struct MockDb; + impl HistoryLookup for MockDb { + fn utxo_id(&self, _: CompressedUtxoId) -> anyhow::Result { + unimplemented!() + } + + fn coin(&self, _: UtxoId) -> anyhow::Result { + unimplemented!() + } + + fn message( + &self, + _: fuel_core_types::fuel_types::Nonce, + ) -> anyhow::Result { + unimplemented!() + } + } + macro_rules! mock_temporal { + ($type:ty) => { + impl TemporalRegistry<$type> for MockDb { + fn read_registry(&self, _key: &RegistryKey) -> anyhow::Result<$type> { + unimplemented!() + } + + fn read_timestamp(&self, _key: &RegistryKey) -> anyhow::Result { + unimplemented!() + } + + fn write_registry( + &mut self, + _key: &RegistryKey, + _value: &$type, + _timestamp: Tai64, + ) -> anyhow::Result<()> { + unimplemented!() + } + + fn registry_index_lookup( + &self, + _value: &$type, + ) -> anyhow::Result> { + unimplemented!() + } + } + + impl EvictorDb<$type> for MockDb { + fn set_latest_assigned_key( + &mut self, + _key: RegistryKey, + ) -> anyhow::Result<()> { + unimplemented!() + } + + fn get_latest_assigned_key(&self) -> anyhow::Result> { + unimplemented!() + } + } + }; + } + mock_temporal!(Address); + mock_temporal!(AssetId); + mock_temporal!(ContractId); + mock_temporal!(ScriptCode); + mock_temporal!(PredicateCode); + + #[tokio::test] + async fn decompress_block_with_unknown_version() { + #[derive(Clone, Serialize, Deserialize)] + enum CompressedBlockWithNewVersions { + V0(crate::CompressedBlockPayloadV0), + NewVersion(u32), + #[serde(untagged)] + Unknown, + } + + // Given + let bad_block = + postcard::to_stdvec(&CompressedBlockWithNewVersions::NewVersion(1234)) + .unwrap(); + + // When + let result: Result = + postcard::from_bytes(&bad_block); + + // Then + let _ = + result.expect_err("should fail to deserialize because of unknown version"); + } +} diff --git a/crates/compression/src/eviction_policy.rs b/crates/compression/src/eviction_policy.rs new file mode 100644 index 00000000000..6343de83d8c --- /dev/null +++ b/crates/compression/src/eviction_policy.rs @@ -0,0 +1,63 @@ +use std::collections::HashSet; + +use fuel_core_types::fuel_compression::RegistryKey; + +use crate::ports::EvictorDb; + +/// Evictor for a single keyspace +#[derive(Debug)] +#[must_use = "Evictor must be committed to the database to persist state"] +pub(crate) struct CacheEvictor { + /// Set of keys that must not be evicted + keep_keys: HashSet, + /// Next key to be used + next_key: RegistryKey, + /// Marker for the keyspace type + _keyspace_marker: std::marker::PhantomData, +} + +impl CacheEvictor { + /// Create new evictor, reading state from the database + pub fn new_from_db( + db: &mut D, + keep_keys: HashSet, + ) -> anyhow::Result + where + D: EvictorDb, + { + let latest_key = db.get_latest_assigned_key()?; + let next_key = if let Some(latest_key) = latest_key { + latest_key.next() + } else { + RegistryKey::ZERO + }; + + Ok(Self { + keep_keys, + next_key, + _keyspace_marker: std::marker::PhantomData, + }) + } + + pub fn next_key(&mut self) -> RegistryKey { + // Pick first key not in the set + // TODO: use a proper algo, maybe LRU? + + debug_assert!(self.keep_keys.len() < 2usize.pow(24).saturating_sub(2)); + + while self.keep_keys.contains(&self.next_key) { + self.next_key = self.next_key.next(); + } + + self.keep_keys.insert(self.next_key); + self.next_key + } + + /// Commit the current state of the evictor to the database + pub fn commit(self, db: &mut D) -> anyhow::Result<()> + where + D: EvictorDb, + { + db.set_latest_assigned_key(self.next_key) + } +} diff --git a/crates/compression/src/lib.rs b/crates/compression/src/lib.rs new file mode 100644 index 00000000000..bd4b0fdcbba --- /dev/null +++ b/crates/compression/src/lib.rs @@ -0,0 +1,155 @@ +#![deny(clippy::arithmetic_side_effects)] +#![deny(clippy::cast_possible_truncation)] +#![deny(unused_crate_dependencies)] +#![deny(warnings)] + +pub mod compress; +pub mod config; +pub mod decompress; +mod eviction_policy; +pub mod ports; +mod registry; + +pub use config::Config; +pub use registry::RegistryKeyspace; + +use fuel_core_types::{ + blockchain::header::PartialBlockHeader, + fuel_tx::CompressedTransaction, +}; +use registry::RegistrationsPerTable; + +/// Compressed block, without the preceding version byte. +#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct CompressedBlockPayloadV0 { + /// Temporal registry insertions + pub registrations: RegistrationsPerTable, + /// Compressed block header + pub header: PartialBlockHeader, + /// Compressed transactions + pub transactions: Vec, +} + +/// Versioned compressed block. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum VersionedCompressedBlock { + V0(CompressedBlockPayloadV0), +} + +impl Default for VersionedCompressedBlock { + fn default() -> Self { + Self::V0(Default::default()) + } +} + +#[cfg(test)] +mod tests { + use fuel_core_compression as _; + use fuel_core_types::{ + blockchain::{ + header::{ + ApplicationHeader, + ConsensusHeader, + }, + primitives::Empty, + }, + fuel_compression::RegistryKey, + tai64::Tai64, + }; + use proptest::prelude::*; + + use super::*; + + fn keyspace() -> impl Strategy { + prop_oneof![ + Just(RegistryKeyspace::Address), + Just(RegistryKeyspace::AssetId), + Just(RegistryKeyspace::ContractId), + Just(RegistryKeyspace::ScriptCode), + Just(RegistryKeyspace::PredicateCode), + ] + } + + proptest! { + /// Serialization for compressed transactions is already tested in fuel-vm, + /// but the rest of the block de/serialization is tested here. + #[test] + fn postcard_roundtrip( + da_height in 0..=u64::MAX, + prev_root in prop::array::uniform32(0..=u8::MAX), + height in 0..=u32::MAX, + consensus_parameters_version in 0..=u32::MAX, + state_transition_bytecode_version in 0..=u32::MAX, + registration_inputs in prop::collection::vec( + (keyspace(), prop::num::u16::ANY, prop::array::uniform32(0..=u8::MAX)).prop_map(|(ks, rk, arr)| { + let k = RegistryKey::try_from(rk as u32).unwrap(); + (ks, k, arr) + }), + 0..123 + ), + ) { + let mut registrations: RegistrationsPerTable = Default::default(); + + for (ks, key, arr) in registration_inputs { + let value_len_limit = (key.as_u32() % 32) as usize; + match ks { + RegistryKeyspace::Address => { + registrations.address.push((key, arr.into())); + } + RegistryKeyspace::AssetId => { + registrations.asset_id.push((key, arr.into())); + } + RegistryKeyspace::ContractId => { + registrations.contract_id.push((key, arr.into())); + } + RegistryKeyspace::ScriptCode => { + registrations.script_code.push((key, arr[..value_len_limit].to_vec().into())); + } + RegistryKeyspace::PredicateCode => { + registrations.predicate_code.push((key, arr[..value_len_limit].to_vec().into())); + } + } + } + + let header = PartialBlockHeader { + application: ApplicationHeader { + da_height: da_height.into(), + consensus_parameters_version, + state_transition_bytecode_version, + generated: Empty, + }, + consensus: ConsensusHeader { + prev_root: prev_root.into(), + height: height.into(), + time: Tai64::UNIX_EPOCH, + generated: Empty + } + }; + let original = CompressedBlockPayloadV0 { + registrations, + header, + transactions: vec![], + }; + + let compressed = postcard::to_allocvec(&original).unwrap(); + let decompressed: CompressedBlockPayloadV0 = + postcard::from_bytes(&compressed).unwrap(); + + let CompressedBlockPayloadV0 { + registrations, + header, + transactions, + } = decompressed; + + assert_eq!(registrations, original.registrations); + + assert_eq!(header.da_height, da_height.into()); + assert_eq!(*header.prev_root(), prev_root.into()); + assert_eq!(*header.height(), height.into()); + assert_eq!(header.consensus_parameters_version, consensus_parameters_version); + assert_eq!(header.state_transition_bytecode_version, state_transition_bytecode_version); + + assert!(transactions.is_empty()); + } + } +} diff --git a/crates/compression/src/ports.rs b/crates/compression/src/ports.rs new file mode 100644 index 00000000000..50b2acd6fe8 --- /dev/null +++ b/crates/compression/src/ports.rs @@ -0,0 +1,121 @@ +//! Ports this service requires to function. + +use fuel_core_types::{ + fuel_compression::RegistryKey, + fuel_tx::{ + Address, + AssetId, + CompressedUtxoId, + UtxoId, + Word, + }, + fuel_types::Nonce, + tai64::Tai64, +}; + +/// Rolling cache for compression. +/// Holds the latest state which can be event sourced from the compressed blocks. +/// The changes done using this trait in a single call to `compress` or `decompress` +/// must be committed atomically, after which block height must be incremented. +pub trait TemporalRegistry { + /// Reads a value from the registry at its current height. + fn read_registry(&self, key: &RegistryKey) -> anyhow::Result; + + /// Reads timestamp of the value from the registry. + fn read_timestamp(&self, key: &RegistryKey) -> anyhow::Result; + + /// Writes a value from to the registry. The timestamp is the time of the block, + /// and it is used for key retention. + fn write_registry( + &mut self, + key: &RegistryKey, + value: &T, + timestamp: Tai64, + ) -> anyhow::Result<()>; + + /// Lookup registry key by the value. + fn registry_index_lookup(&self, value: &T) -> anyhow::Result>; +} + +impl TemporalRegistry for &mut D +where + D: TemporalRegistry, +{ + fn read_registry(&self, key: &RegistryKey) -> anyhow::Result { + >::read_registry(self, key) + } + + fn read_timestamp(&self, key: &RegistryKey) -> anyhow::Result { + >::read_timestamp(self, key) + } + + fn write_registry( + &mut self, + key: &RegistryKey, + value: &T, + timestamp: Tai64, + ) -> anyhow::Result<()> { + >::write_registry(self, key, value, timestamp) + } + + fn registry_index_lookup(&self, value: &T) -> anyhow::Result> { + >::registry_index_lookup(self, value) + } +} + +/// Lookup for UTXO pointers used for compression. +pub trait UtxoIdToPointer { + fn lookup(&self, utxo_id: UtxoId) -> anyhow::Result; +} + +impl UtxoIdToPointer for &mut D +where + D: UtxoIdToPointer, +{ + fn lookup(&self, utxo_id: UtxoId) -> anyhow::Result { + ::lookup(self, utxo_id) + } +} + +/// Lookup for history of UTXOs and messages, used for decompression. +pub trait HistoryLookup { + fn utxo_id(&self, c: CompressedUtxoId) -> anyhow::Result; + fn coin(&self, utxo_id: UtxoId) -> anyhow::Result; + fn message(&self, nonce: Nonce) -> anyhow::Result; +} + +/// Information about a coin. +#[derive(Debug, Clone)] +pub struct CoinInfo { + pub owner: Address, + pub amount: u64, + pub asset_id: AssetId, +} + +/// Information about a message. +#[derive(Debug, Clone)] +pub struct MessageInfo { + pub sender: Address, + pub recipient: Address, + pub amount: Word, + pub data: Vec, +} + +/// Evictor registry to keep track of the latest used key for the type `T`. +pub trait EvictorDb { + fn get_latest_assigned_key(&self) -> anyhow::Result>; + fn set_latest_assigned_key(&mut self, key: RegistryKey) -> anyhow::Result<()>; +} + +impl EvictorDb for &mut D +where + D: EvictorDb, +{ + fn get_latest_assigned_key(&self) -> anyhow::Result> { + >::get_latest_assigned_key(self) + } + + fn set_latest_assigned_key(&mut self, key: RegistryKey) -> anyhow::Result<()> { + >::set_latest_assigned_key(self, key) + } +} diff --git a/crates/compression/src/registry.rs b/crates/compression/src/registry.rs new file mode 100644 index 00000000000..0bf1e3a5967 --- /dev/null +++ b/crates/compression/src/registry.rs @@ -0,0 +1,119 @@ +use crate::ports::{ + EvictorDb, + TemporalRegistry, +}; +use fuel_core_types::{ + fuel_compression::RegistryKey, + fuel_tx::{ + input::PredicateCode, + Address, + AssetId, + ContractId, + ScriptCode, + }, + tai64::Tai64, +}; + +macro_rules! tables { + ($($ident:ty: $type:ty),*) => { paste::paste! { + #[doc = "RegistryKey namespaces"] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, strum_macros::EnumCount)] + pub enum RegistryKeyspace { + $( + [<$type>], + )* + } + + #[doc = "A value for each keyspace"] + #[derive(Debug, Clone, PartialEq, Eq, Default)] + pub struct PerRegistryKeyspace { + $(pub $ident: T,)* + } + impl core::ops::Index for PerRegistryKeyspace { + type Output = T; + + fn index(&self, index: RegistryKeyspace) -> &Self::Output { + match index { + $( + RegistryKeyspace::[<$type>] => &self.$ident, + )* + } + } + } + impl core::ops::IndexMut for PerRegistryKeyspace { + fn index_mut(&mut self, index: RegistryKeyspace) -> &mut Self::Output { + match index { + $( + RegistryKeyspace::[<$type>] => &mut self.$ident, + )* + } + } + } + + #[doc = "The set of registrations for each table, as used in the compressed block header"] + #[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] + pub struct RegistrationsPerTable { + $(pub $ident: Vec<(RegistryKey, $type)>,)* + } + + pub trait TemporalRegistryAll + where + $(Self: TemporalRegistry<$type>,)* + {} + + impl TemporalRegistryAll for T + where + $(T: TemporalRegistry<$type>,)* + {} + + pub trait EvictorDbAll + where + $(Self: EvictorDb<$type>,)* + {} + + impl EvictorDbAll for T + where + $(T: EvictorDb<$type>,)* + {} + + + impl RegistrationsPerTable { + pub(crate) fn write_to_registry(&self, registry: &mut R, timestamp: Tai64) -> anyhow::Result<()> + where + R: TemporalRegistryAll + { + $( + for (key, value) in self.$ident.iter() { + registry.write_registry(key, value, timestamp)?; + } + )* + + Ok(()) + } + } + }}; +} + +tables!( + address: Address, + asset_id: AssetId, + contract_id: ContractId, + script_code: ScriptCode, + predicate_code: PredicateCode +); + +// TODO: move inside the macro when this stabilizes: https://github.com/rust-lang/rust/pull/122808 +#[cfg(any(test, feature = "test-helpers"))] +impl rand::prelude::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> RegistryKeyspace { + use strum::EnumCount; + match rng.gen_range(0..RegistryKeyspace::COUNT) { + 0 => RegistryKeyspace::Address, + 1 => RegistryKeyspace::AssetId, + 2 => RegistryKeyspace::ContractId, + 3 => RegistryKeyspace::ScriptCode, + 4 => RegistryKeyspace::PredicateCode, + _ => unreachable!("New keyspace is added but not supported here"), + } + } +} diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index 40749a3814e..29efb372a25 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -12,7 +12,7 @@ version = { workspace = true } [dependencies] anyhow = { workspace = true } -async-graphql = { version = "7.0.6", features = [ +async-graphql = { version = "7.0.11", features = [ "playground", "tracing", ], default-features = false } @@ -22,6 +22,7 @@ clap = { workspace = true, features = ["derive"] } derive_more = { version = "0.99" } enum-iterator = { workspace = true } fuel-core-chain-config = { workspace = true, features = ["std"] } +fuel-core-compression = { workspace = true } fuel-core-consensus-module = { workspace = true } fuel-core-database = { workspace = true } fuel-core-executor = { workspace = true, features = ["std"] } @@ -45,6 +46,7 @@ indicatif = { workspace = true, default-features = true } itertools = { workspace = true } num_cpus = { version = "1.16.0", optional = true } parking_lot = { workspace = true } +paste = { workspace = true } postcard = { workspace = true, optional = true } rand = { workspace = true } rocksdb = { version = "0.21", default-features = false, features = [ @@ -61,7 +63,8 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tokio-rayon = { workspace = true } tokio-stream = { workspace = true, features = ["sync"] } tokio-util = { workspace = true } -tower-http = { version = "0.3", features = ["set-header", "trace", "timeout"] } +tower = { version = "0.4", features = ["limit"] } +tower-http = { version = "0.4", features = ["set-header", "trace", "timeout"] } tracing = { workspace = true } uuid = { version = "1.1", features = ["v4"] } @@ -96,6 +99,7 @@ test-helpers = [ "fuel-core-p2p?/test-helpers", "fuel-core-storage/test-helpers", "fuel-core-chain-config/test-helpers", + "fuel-core-compression/test-helpers", "fuel-core-txpool/test-helpers", "fuel-core-services/test-helpers", "fuel-core-importer/test-helpers", diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index a5f527969fd..83250fbcf30 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -312,17 +312,46 @@ impl CombinedDatabase { Ok(()) } + /// This function is fundamentally different from `rollback_to` in that it + /// will rollback the off-chain/gas-price databases if they are ahead of the + /// on-chain database. If they don't have a height or are behind the on-chain + /// we leave it to the caller to decide how to bring them up to date. + /// We don't rollback the on-chain database as it is the source of truth. + /// The target height of the rollback is the latest height of the on-chain database. pub fn sync_aux_db_heights(&self, shutdown_listener: &mut S) -> anyhow::Result<()> where S: ShutdownListener, { - if let Some(on_chain_height) = self.on_chain().latest_height_from_metadata()? { - // todo(https://github.com/FuelLabs/fuel-core/issues/2239): This is a temporary fix - let res = self.rollback_to(on_chain_height, shutdown_listener); - if res.is_err() { - tracing::warn!("Failed to rollback auxiliary databases to on-chain database height: {:?}", res); + while !shutdown_listener.is_cancelled() { + let on_chain_height = match self.on_chain().latest_height_from_metadata()? { + Some(height) => height, + None => break, // Exit loop if on-chain height is None + }; + + let off_chain_height = self.off_chain().latest_height_from_metadata()?; + let gas_price_height = self.gas_price().latest_height_from_metadata()?; + + // Handle off-chain rollback if necessary + if let Some(off_height) = off_chain_height { + if off_height > on_chain_height { + self.off_chain().rollback_last_block()?; + } } - }; + + // Handle gas price rollback if necessary + if let Some(gas_height) = gas_price_height { + if gas_height > on_chain_height { + self.gas_price().rollback_last_block()?; + } + } + + // If both off-chain and gas price heights are synced, break + if off_chain_height.map_or(true, |h| h <= on_chain_height) + && gas_price_height.map_or(true, |h| h <= on_chain_height) + { + break; + } + } Ok(()) } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 851ebd08b2a..ca3607275b6 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -19,17 +19,18 @@ use crate::{ }, generic_database::GenericDatabase, in_memory::memory_store::MemoryStore, - ChangesIterator, ColumnType, IterableKeyValueView, KeyValueView, }, }; use fuel_core_chain_config::TableEntry; +use fuel_core_gas_price_service::common::fuel_core_storage_adapter::storage::GasPriceMetadata; use fuel_core_services::SharedMutex; use fuel_core_storage::{ self, iter::{ + changes_iterator::ChangesIterator, IterDirection, IterableTable, IteratorOverTable, @@ -75,7 +76,6 @@ use crate::state::{ }, rocks_db::RocksDb, }; -use fuel_core_gas_price_service::common::fuel_core_storage_adapter::storage::GasPriceMetadata; #[cfg(feature = "rocksdb")] use std::path::Path; @@ -401,7 +401,7 @@ fn commit_changes_with_height_update( database: &mut Database, changes: Changes, heights_lookup: impl Fn( - &ChangesIterator, + &ChangesIterator, ) -> StorageResult>, ) -> StorageResult<()> where @@ -411,7 +411,7 @@ where StorageMutate, Error = StorageError>, { // Gets the all new heights from the `changes` - let iterator = ChangesIterator::::new(&changes); + let iterator = ChangesIterator::::new(&changes); let new_heights = heights_lookup(&iterator)?; // Changes for each block should be committed separately. diff --git a/crates/fuel-core/src/database/coin.rs b/crates/fuel-core/src/database/coin.rs index 4c94d202bd7..488e0a90456 100644 --- a/crates/fuel-core/src/database/coin.rs +++ b/crates/fuel-core/src/database/coin.rs @@ -18,11 +18,11 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsRef, }; -use fuel_core_txpool::types::TxId; use fuel_core_types::{ entities::coins::coin::CompressedCoin, fuel_tx::{ Address, + TxId, UtxoId, }, }; diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 8d1a1fd37f2..412d13c18cf 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -3,9 +3,6 @@ #[allow(non_snake_case)] #[cfg(test)] mod tests { - - use std::sync::Mutex; - use crate as fuel_core; use fuel_core::database::Database; use fuel_core_executor::{ @@ -127,6 +124,7 @@ mod tests { ExecutableTransaction, MemoryInstance, }, + predicate::EmptyStorage, script_with_data_offset, util::test_helpers::TestBuilder as TxBuilder, Call, @@ -153,6 +151,7 @@ mod tests { Rng, SeedableRng, }; + use std::sync::Mutex; #[derive(Clone, Debug, Default)] struct Config { @@ -355,7 +354,7 @@ mod tests { let tx = TransactionBuilder::create(contract_code.into(), salt, Default::default()) - .add_random_fee_input() + .add_fee_input() .add_output(Output::contract_created(contract_id, state_root)) .finalize(); (tx, contract_id) @@ -1478,7 +1477,7 @@ mod tests { header: Default::default(), transactions: vec![TransactionBuilder::script(vec![], vec![]) .max_fee_limit(100_000_000) - .add_random_fee_input() + .add_fee_input() .script_gas_limit(0) .tip(123) .finalize_as_transaction()], @@ -1494,7 +1493,7 @@ mod tests { for i in 0..10 { let tx = TransactionBuilder::script(vec![], vec![]) .max_fee_limit(100_000_000) - .add_random_fee_input() + .add_fee_input() .script_gas_limit(0) .tip(i * 100) .finalize_as_transaction(); @@ -1502,9 +1501,10 @@ mod tests { } let mut config: Config = Default::default(); // Each TX consumes `tx_gas_usage` gas and so set the block gas limit to execute only 9 transactions. + let block_gas_limit = tx_gas_usage * 9; config .consensus_parameters - .set_block_gas_limit(tx_gas_usage * 9); + .set_block_gas_limit(block_gas_limit); let mut executor = create_executor(Default::default(), config); let block = PartialFuelBlock { @@ -1520,7 +1520,14 @@ mod tests { // Then assert_eq!(skipped_transactions.len(), 1); - assert_eq!(skipped_transactions[0].1, ExecutorError::GasOverflow); + assert_eq!( + skipped_transactions[0].1, + ExecutorError::GasOverflow( + "Transaction cannot fit in remaining gas limit: (0).".into(), + *tx_gas_usage, + 0 + ) + ); } #[test] @@ -1533,7 +1540,7 @@ mod tests { // The test checks that execution for the block with transactions [tx1, tx2, tx3] skips // transaction `tx1` and produce a block [tx2, tx3] with the expected order. let tx1 = TransactionBuilder::script(vec![], vec![]) - .add_random_fee_input() + .add_fee_input() .script_gas_limit(1000000) .tip(1000000) .finalize_as_transaction(); @@ -2633,7 +2640,7 @@ mod tests { .collect(), vec![], ) - .add_random_fee_input() + .add_fee_input() .script_gas_limit(1000000) .finalize_as_transaction(); @@ -2697,7 +2704,7 @@ mod tests { .collect(), vec![], ) - .add_random_fee_input() + .add_fee_input() .script_gas_limit(1000000) .finalize_as_transaction(); @@ -2886,6 +2893,7 @@ mod tests { tx.estimate_predicates( &consensus_parameters.clone().into(), MemoryInstance::new(), + &EmptyStorage, ) .unwrap(); let db = &mut Database::default(); @@ -2954,6 +2962,7 @@ mod tests { tx.estimate_predicates( &cheap_consensus_parameters.clone().into(), MemoryInstance::new(), + &EmptyStorage, ) .unwrap(); @@ -3100,16 +3109,17 @@ mod tests { #[cfg(feature = "relayer")] mod relayer { use super::*; - use crate::{ - database::database_description::{ - on_chain::OnChain, - relayer::Relayer, - }, - state::ChangesIterator, + use crate::database::database_description::{ + on_chain::OnChain, + relayer::Relayer, }; use fuel_core_relayer::storage::EventsHistory; use fuel_core_storage::{ - iter::IteratorOverTable, + column::Column, + iter::{ + changes_iterator::ChangesIterator, + IteratorOverTable, + }, tables::FuelBlocks, StorageAsMut, }; @@ -3254,7 +3264,7 @@ mod tests { let (result, changes) = producer.produce_without_commit(block.into())?.into(); // Then - let view = ChangesIterator::::new(&changes); + let view = ChangesIterator::::new(&changes); assert_eq!( view.iter_all::(None).count() as u64, block_da_height - genesis_da_height @@ -3863,7 +3873,7 @@ mod tests { .into(); // Then - let view = ChangesIterator::::new(&changes); + let view = ChangesIterator::::new(&changes); assert!(result.skipped_transactions.is_empty()); assert_eq!(view.iter_all::(None).count() as u64, 0); } @@ -3905,7 +3915,7 @@ mod tests { .into(); // Then - let view = ChangesIterator::::new(&changes); + let view = ChangesIterator::::new(&changes); assert!(result.skipped_transactions.is_empty()); assert_eq!(view.iter_all::(None).count() as u64, 0); assert_eq!(result.events.len(), 2); diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index c3fc9abb991..46e2dc0d676 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -8,6 +8,7 @@ use std::{ }; pub mod api_service; +mod da_compression; pub mod database; pub(crate) mod metrics_extension; pub mod ports; @@ -21,6 +22,8 @@ pub struct ServiceConfig { pub max_queries_depth: usize, pub max_queries_complexity: usize, pub max_queries_recursive_depth: usize, + pub max_queries_directives: usize, + pub max_concurrent_queries: usize, pub request_body_bytes_limit: usize, /// Time to wait after submitting a query before debug info will be logged about query. pub query_log_threshold_time: Duration, @@ -36,29 +39,43 @@ pub struct Costs { pub submit: usize, pub submit_and_await: usize, pub status_change: usize, - pub raw_payload: usize, pub storage_read: usize, + pub tx_get: usize, + pub tx_status_read: usize, + pub tx_raw_payload: usize, + pub block_header: usize, + pub block_transactions: usize, + pub block_transactions_ids: usize, pub storage_iterator: usize, pub bytecode_read: usize, + pub state_transition_bytecode_read: usize, + pub da_compressed_block_read: usize, } pub const QUERY_COSTS: Costs = Costs { // balance_query: 4000, - balance_query: 10001, - coins_to_spend: 10001, + balance_query: 40001, + coins_to_spend: 40001, // get_peers: 2000, - get_peers: 10001, + get_peers: 40001, // estimate_predicates: 3000, - estimate_predicates: 10001, - dry_run: 3000, + estimate_predicates: 40001, + dry_run: 12000, // submit: 5000, - submit: 10001, - submit_and_await: 10001, - status_change: 10001, - raw_payload: 10, - storage_read: 10, + submit: 40001, + submit_and_await: 40001, + status_change: 40001, + storage_read: 40, + tx_get: 50, + tx_status_read: 50, + tx_raw_payload: 150, + block_header: 150, + block_transactions: 1500, + block_transactions_ids: 50, storage_iterator: 100, - bytecode_read: 2000, + bytecode_read: 8000, + state_transition_bytecode_read: 76_000, + da_compressed_block_read: 4000, }; #[derive(Clone, Debug)] @@ -68,7 +85,7 @@ pub struct Config { pub debug: bool, pub vm_backtrace: bool, pub max_tx: usize, - pub max_txpool_depth: usize, + pub max_txpool_dependency_chain_length: usize, pub chain_name: String, } diff --git a/crates/fuel-core/src/graphql_api/api_service.rs b/crates/fuel-core/src/graphql_api/api_service.rs index 3305c98828a..d1ccbce203e 100644 --- a/crates/fuel-core/src/graphql_api/api_service.rs +++ b/crates/fuel-core/src/graphql_api/api_service.rs @@ -75,6 +75,7 @@ use std::{ pin::Pin, }; use tokio_stream::StreamExt; +use tower::limit::ConcurrencyLimitLayer; use tower_http::{ set_header::SetResponseHeaderLayer, timeout::TimeoutLayer, @@ -196,12 +197,14 @@ where let combined_read_database = ReadDatabase::new(genesis_block_height, on_database, off_database); let request_timeout = config.config.api_request_timeout; + let concurrency_limit = config.config.max_concurrent_queries; let body_limit = config.config.request_body_bytes_limit; let schema = schema .limit_complexity(config.config.max_queries_complexity) .limit_depth(config.config.max_queries_depth) .limit_recursive_depth(config.config.max_queries_recursive_depth) + .limit_directives(config.config.max_queries_directives) .extension(MetricsExtension::new( config.config.query_log_threshold_time, )) @@ -220,7 +223,12 @@ where let router = Router::new() .route("/v1/playground", get(graphql_playground)) - .route("/v1/graphql", post(graphql_handler).options(ok)) + .route( + "/v1/graphql", + post(graphql_handler) + .layer(ConcurrencyLimitLayer::new(concurrency_limit)) + .options(ok), + ) .route( "/v1/graphql-sub", post(graphql_subscription_handler).options(ok), diff --git a/crates/fuel-core/src/graphql_api/da_compression.rs b/crates/fuel-core/src/graphql_api/da_compression.rs new file mode 100644 index 00000000000..e9d11d1c22e --- /dev/null +++ b/crates/fuel-core/src/graphql_api/da_compression.rs @@ -0,0 +1,212 @@ +use crate::fuel_core_graphql_api::{ + ports::worker::OffChainDatabaseTransaction, + storage::da_compression::{ + evictor_cache::MetadataKey, + timestamps::{ + TimestampKey, + TimestampKeyspace, + }, + *, + }, +}; +use fuel_core_compression::{ + compress::compress, + config::Config, + ports::{ + EvictorDb, + TemporalRegistry, + UtxoIdToPointer, + }, +}; +use fuel_core_storage::{ + not_found, + StorageAsMut, + StorageAsRef, +}; +use fuel_core_types::{ + blockchain::block::Block, + fuel_tx::{ + input::PredicateCode, + Address, + AssetId, + ContractId, + ScriptCode, + }, + services::executor::Event, + tai64::Tai64, +}; +use futures::FutureExt; + +/// Performs DA compression for a block and stores it in the database. +pub fn da_compress_block( + config: Config, + block: &Block, + block_events: &[Event], + db_tx: &mut T, +) -> anyhow::Result<()> +where + T: OffChainDatabaseTransaction, +{ + let compressed = compress( + config, + CompressTx { + db_tx, + block_events, + }, + block, + ) + .now_or_never() + .expect("The current implementation resolved all futures instantly")?; + + db_tx + .storage_as_mut::() + .insert(&block.header().consensus().height, &compressed)?; + + Ok(()) +} + +struct CompressTx<'a, Tx> { + db_tx: &'a mut Tx, + block_events: &'a [Event], +} + +macro_rules! impl_temporal_registry { + ($type:ty) => { paste::paste! { + impl<'a, Tx> TemporalRegistry<$type> for CompressTx<'a, Tx> + where + Tx: OffChainDatabaseTransaction, + { + fn read_registry( + &self, + key: &fuel_core_types::fuel_compression::RegistryKey, + ) -> anyhow::Result<$type> { + Ok(self + .db_tx + .storage_as_ref::<[< DaCompressionTemporalRegistry $type >]>() + .get(key)? + .ok_or(not_found!([< DaCompressionTemporalRegistry $type>]))? + .into_owned()) + } + + fn read_timestamp( + &self, + key: &fuel_core_types::fuel_compression::RegistryKey, + ) -> anyhow::Result { + Ok(self + .db_tx + .storage_as_ref::<[< DaCompressionTemporalRegistryTimestamps >]>() + .get(&TimestampKey { + keyspace: TimestampKeyspace::$type, + key: *key, + })? + .ok_or(not_found!(DaCompressionTemporalRegistryTimestamps))? + .into_owned()) + } + + fn write_registry( + &mut self, + key: &fuel_core_types::fuel_compression::RegistryKey, + value: &$type, + timestamp: Tai64, + ) -> anyhow::Result<()> { + // Write the actual value + let old_value = self.db_tx + .storage_as_mut::<[< DaCompressionTemporalRegistry $type >]>() + .replace(key, value)?; + + // Remove the overwritten value from index, if any + if let Some(old_value) = old_value { + let old_reverse_key = (&old_value).into(); + self.db_tx + .storage_as_mut::() + .remove(&old_reverse_key)?; + } + + // Add the new value to the index + let reverse_key = value.into(); + self.db_tx + .storage_as_mut::() + .insert(&reverse_key, key)?; + + // Update the timestamp + self.db_tx + .storage_as_mut::() + .insert(&TimestampKey { keyspace: TimestampKeyspace::$type, key: *key }, ×tamp)?; + + Ok(()) + } + + fn registry_index_lookup( + &self, + value: &$type, + ) -> anyhow::Result> + { + let reverse_key = value.into(); + Ok(self + .db_tx + .storage_as_ref::() + .get(&reverse_key)? + .map(|v| v.into_owned())) + } + } + + impl<'a, Tx> EvictorDb<$type> for CompressTx<'a, Tx> + where + Tx: OffChainDatabaseTransaction, + { + fn set_latest_assigned_key( + &mut self, + key: fuel_core_types::fuel_compression::RegistryKey, + ) -> anyhow::Result<()> { + self.db_tx + .storage_as_mut::() + .insert(&MetadataKey::$type, &key)?; + Ok(()) + } + + fn get_latest_assigned_key( + &self, + ) -> anyhow::Result> { + Ok(self + .db_tx + .storage_as_ref::() + .get(&MetadataKey::$type)? + .map(|v| v.into_owned()) + ) + } + } + + }}; +} + +impl_temporal_registry!(Address); +impl_temporal_registry!(AssetId); +impl_temporal_registry!(ContractId); +impl_temporal_registry!(ScriptCode); +impl_temporal_registry!(PredicateCode); + +impl<'a, Tx> UtxoIdToPointer for CompressTx<'a, Tx> +where + Tx: OffChainDatabaseTransaction, +{ + fn lookup( + &self, + utxo_id: fuel_core_types::fuel_tx::UtxoId, + ) -> anyhow::Result { + for event in self.block_events { + match event { + Event::CoinCreated(coin) | Event::CoinConsumed(coin) + if coin.utxo_id == utxo_id => + { + let output_index = coin.utxo_id.output_index(); + return Ok(fuel_core_types::fuel_tx::CompressedUtxoId { + tx_pointer: coin.tx_pointer, + output_index, + }); + } + _ => {} + } + } + anyhow::bail!("UtxoId not found in the block events"); + } +} diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index ca1dd6ca972..418e14c1318 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -23,12 +23,11 @@ use fuel_core_storage::{ Error as StorageError, IsNotFound, Mappable, + PredicateStorageRequirements, Result as StorageResult, StorageInspect, -}; -use fuel_core_txpool::types::{ - ContractId, - TxId, + StorageRead, + StorageSize, }; use fuel_core_types::{ blockchain::{ @@ -50,15 +49,19 @@ use fuel_core_types::{ Address, AssetId, Bytes32, + ContractId, Salt, Transaction, + TxId, TxPointer, UtxoId, }, fuel_types::{ + BlobId, BlockHeight, Nonce, }, + fuel_vm::BlobData, services::{ graphql_api::ContractBalance, txpool::TransactionStatus, @@ -69,6 +72,8 @@ use std::{ sync::Arc, }; +use super::ports::DatabaseDaCompressedBlocks; + mod arc_wrapper; /// The on-chain view of the database used by the [`ReadView`] to fetch on-chain data. @@ -210,6 +215,16 @@ impl DatabaseBlocks for ReadView { } } +impl DatabaseDaCompressedBlocks for ReadView { + fn da_compressed_block(&self, id: &BlockHeight) -> StorageResult> { + self.off_chain.da_compressed_block(id) + } + + fn latest_height(&self) -> StorageResult { + self.on_chain.latest_height() + } +} + impl StorageInspect for ReadView where M: Mappable, @@ -226,6 +241,28 @@ where } } +impl StorageSize for ReadView { + fn size_of_value(&self, key: &BlobId) -> Result, Self::Error> { + self.on_chain.size_of_value(key) + } +} + +impl StorageRead for ReadView { + fn read(&self, key: &BlobId, buf: &mut [u8]) -> Result, Self::Error> { + self.on_chain.read(key, buf) + } + + fn read_alloc(&self, key: &BlobId) -> Result>, Self::Error> { + self.on_chain.read_alloc(key) + } +} + +impl PredicateStorageRequirements for ReadView { + fn storage_error_to_string(error: Self::Error) -> String { + error.to_string() + } +} + impl DatabaseMessages for ReadView { fn all_messages( &self, @@ -286,6 +323,10 @@ impl OffChainDatabase for ReadView { self.off_chain.block_height(block_id) } + fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult> { + self.off_chain.da_compressed_block(height) + } + fn tx_status(&self, tx_id: &TxId) -> StorageResult { self.off_chain.tx_status(tx_id) } diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 2dcd596f2eb..077a48d1637 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -17,8 +17,9 @@ use fuel_core_storage::{ Error as StorageError, Result as StorageResult, StorageInspect, + StorageRead, }; -use fuel_core_txpool::service::TxStatusMessage; +use fuel_core_txpool::TxStatusMessage; use fuel_core_types::{ blockchain::{ block::CompressedBlock, @@ -57,10 +58,7 @@ use fuel_core_types::{ executor::TransactionExecutionStatus, graphql_api::ContractBalance, p2p::PeerInfo, - txpool::{ - InsertionResult, - TransactionStatus, - }, + txpool::TransactionStatus, }, tai64::Tai64, }; @@ -69,6 +67,8 @@ use std::sync::Arc; pub trait OffChainDatabase: Send + Sync { fn block_height(&self, block_id: &BlockId) -> StorageResult; + fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult>; + fn tx_status(&self, tx_id: &TxId) -> StorageResult; fn owned_coins_ids( @@ -121,7 +121,7 @@ pub trait OnChainDatabase: + DatabaseBlocks + DatabaseMessages + StorageInspect - + StorageInspect + + StorageRead + StorageInspect + StorageInspect + DatabaseContracts @@ -150,6 +150,14 @@ pub trait DatabaseBlocks { fn consensus(&self, id: &BlockHeight) -> StorageResult; } +/// Trait that specifies all the getters required for DA compressed blocks. +pub trait DatabaseDaCompressedBlocks { + /// Get a DA compressed block by its height. + fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult>; + + fn latest_height(&self) -> StorageResult; +} + /// Trait that specifies all the getters required for messages. pub trait DatabaseMessages: StorageInspect { fn all_messages( @@ -188,14 +196,11 @@ pub trait DatabaseChain { #[async_trait] pub trait TxPoolPort: Send + Sync { - fn transaction(&self, id: TxId) -> Option; + async fn transaction(&self, id: TxId) -> anyhow::Result>; - fn submission_time(&self, id: TxId) -> Option; + async fn submission_time(&self, id: TxId) -> anyhow::Result>; - async fn insert( - &self, - txs: Vec>, - ) -> Vec>; + async fn insert(&self, txs: Transaction) -> anyhow::Result<()>; fn tx_update_subscribe( &self, @@ -268,6 +273,7 @@ pub mod worker { }, }, graphql_api::storage::{ + da_compression::*, old::{ OldFuelBlockConsensus, OldFuelBlocks, @@ -321,6 +327,15 @@ pub mod worker { + StorageMutate + StorageMutate + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate + + StorageMutate { fn record_tx_id_owner( &mut self, diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs index 1b77c07cbdc..8f8cfcd1f19 100644 --- a/crates/fuel-core/src/graphql_api/storage.rs +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -39,6 +39,7 @@ use statistic::StatisticTable; pub mod blocks; pub mod coins; pub mod contracts; +pub mod da_compression; pub mod messages; pub mod old; pub mod statistic; @@ -93,6 +94,25 @@ pub enum Column { /// Existence of a key in this column means that the message has been spent. /// See [`SpentMessages`](messages::SpentMessages) SpentMessages = 13, + /// DA compression and postcard serialized blocks. + /// See [`DaCompressedBlocks`](da_compression::DaCompressedBlocks) + DaCompressedBlocks = 14, + /// See [`DaCompressionTemporalRegistryIndex`](da_compression::DaCompressionTemporalRegistryIndex) + DaCompressionTemporalRegistryIndex = 15, + /// See [`DaCompressionTemporalRegistryTimestamps`](da_compression::DaCompressionTemporalRegistryTimestamps) + DaCompressionTemporalRegistryTimestamps = 16, + /// See [`DaCompressionTemporalRegistryEvictorCache`](da_compression::DaCompressionTemporalRegistryEvictorCache) + DaCompressionTemporalRegistryEvictorCache = 17, + /// See [`DaCompressionTemporalRegistryAddress`](da_compression::DaCompressionTemporalRegistryAddress) + DaCompressionTemporalRegistryAddress = 18, + /// See [`DaCompressionTemporalRegistryAssetId`](da_compression::DaCompressionTemporalRegistryAssetId) + DaCompressionTemporalRegistryAssetId = 19, + /// See [`DaCompressionTemporalRegistryContractId`](da_compression::DaCompressionTemporalRegistryContractId) + DaCompressionTemporalRegistryContractId = 20, + /// See [`DaCompressionTemporalRegistryScriptCode`](da_compression::DaCompressionTemporalRegistryScriptCode) + DaCompressionTemporalRegistryScriptCode = 21, + /// See [`DaCompressionTemporalRegistryPredicateCode`](da_compression::DaCompressionTemporalRegistryPredicateCode) + DaCompressionTemporalRegistryPredicateCode = 22, } impl Column { diff --git a/crates/fuel-core/src/graphql_api/storage/coins.rs b/crates/fuel-core/src/graphql_api/storage/coins.rs index 58c8a23958d..42d22ba94ec 100644 --- a/crates/fuel-core/src/graphql_api/storage/coins.rs +++ b/crates/fuel-core/src/graphql_api/storage/coins.rs @@ -8,9 +8,9 @@ use fuel_core_storage::{ structured_storage::TableWithBlueprint, Mappable, }; -use fuel_core_txpool::types::TxId; use fuel_core_types::fuel_tx::{ Address, + TxId, UtxoId, }; diff --git a/crates/fuel-core/src/graphql_api/storage/contracts.rs b/crates/fuel-core/src/graphql_api/storage/contracts.rs index 27056502b61..51b4168ba14 100644 --- a/crates/fuel-core/src/graphql_api/storage/contracts.rs +++ b/crates/fuel-core/src/graphql_api/storage/contracts.rs @@ -7,8 +7,10 @@ use fuel_core_storage::{ structured_storage::TableWithBlueprint, Mappable, }; -use fuel_core_txpool::types::ContractId; -use fuel_core_types::entities::contract::ContractsInfoType; +use fuel_core_types::{ + entities::contract::ContractsInfoType, + fuel_tx::ContractId, +}; /// Contract info pub struct ContractsInfo; diff --git a/crates/fuel-core/src/graphql_api/storage/da_compression.rs b/crates/fuel-core/src/graphql_api/storage/da_compression.rs new file mode 100644 index 00000000000..54b21073ed6 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/da_compression.rs @@ -0,0 +1,200 @@ +use self::{ + evictor_cache::MetadataKey, + predicate_code_codec::PredicateCodeCodec, + reverse_key::ReverseKey, + script_code_codec::ScriptCodeCodec, + timestamps::TimestampKey, +}; +use fuel_core_compression::VersionedCompressedBlock; +use fuel_core_storage::{ + blueprint::plain::Plain, + codec::{ + postcard::Postcard, + primitive::Primitive, + raw::Raw, + }, + structured_storage::TableWithBlueprint, + Mappable, +}; +use fuel_core_types::{ + fuel_compression::RegistryKey, + fuel_tx::{ + input::PredicateCode, + Address, + AssetId, + ContractId, + ScriptCode, + }, + fuel_types::BlockHeight, + tai64::Tai64, +}; + +pub mod evictor_cache; +pub mod predicate_code_codec; +pub mod reverse_key; +pub mod script_code_codec; +pub mod timestamps; + +/// The table for the compressed blocks sent to DA. +pub struct DaCompressedBlocks; + +impl Mappable for DaCompressedBlocks { + type Key = Self::OwnedKey; + type OwnedKey = BlockHeight; + type Value = Self::OwnedValue; + type OwnedValue = VersionedCompressedBlock; +} + +impl TableWithBlueprint for DaCompressedBlocks { + type Blueprint = Plain, Postcard>; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::DaCompressedBlocks + } +} + +/// Mapping from the type to the registry key in the temporal registry. +pub struct DaCompressionTemporalRegistryIndex; + +impl Mappable for DaCompressionTemporalRegistryIndex { + type Key = Self::OwnedKey; + type OwnedKey = ReverseKey; + type Value = Self::OwnedValue; + type OwnedValue = RegistryKey; +} + +impl TableWithBlueprint for DaCompressionTemporalRegistryIndex { + // TODO: Use Raw codec for value instead of Postcard + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::DaCompressionTemporalRegistryIndex + } +} + +/// This table keeps track of last written timestamp for each key, +/// so that we can keep track of expiration. +pub struct DaCompressionTemporalRegistryTimestamps; + +impl Mappable for DaCompressionTemporalRegistryTimestamps { + type Key = Self::OwnedKey; + type OwnedKey = TimestampKey; + type Value = Self::OwnedValue; + type OwnedValue = Tai64; +} + +impl TableWithBlueprint for DaCompressionTemporalRegistryTimestamps { + // TODO: Use Raw codec for value instead of Postcard + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::DaCompressionTemporalRegistryTimestamps + } +} + +/// This table is used to hold "next key to evict" for each keyspace. +/// In the future we'll likely switch to use LRU or something, in which +/// case this table can be repurposed. +pub struct DaCompressionTemporalRegistryEvictorCache; + +impl Mappable for DaCompressionTemporalRegistryEvictorCache { + type Key = Self::OwnedKey; + type OwnedKey = MetadataKey; + type Value = Self::OwnedValue; + type OwnedValue = RegistryKey; +} + +impl TableWithBlueprint for DaCompressionTemporalRegistryEvictorCache { + // TODO: Use Raw codec for value instead of Postcard + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::DaCompressionTemporalRegistryEvictorCache + } +} + +macro_rules! temporal_registry { + ($type:ty, $code:ty) => { + paste::paste! { + pub struct [< DaCompressionTemporalRegistry $type >]; + + impl Mappable for [< DaCompressionTemporalRegistry $type >] { + type Key = Self::OwnedKey; + type OwnedKey = RegistryKey; + type Value = Self::OwnedValue; + type OwnedValue = $type; + } + + impl TableWithBlueprint for [< DaCompressionTemporalRegistry $type >] { + // TODO: Use Raw codec for value instead of Postcard + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::[< DaCompressionTemporalRegistry $type >] + } + } + + + #[cfg(test)] + fuel_core_storage::basic_storage_tests!( + [< DaCompressionTemporalRegistry $type >], + RegistryKey::ZERO, + <[< DaCompressionTemporalRegistry $type >] as Mappable>::Value::default(), + <[< DaCompressionTemporalRegistry $type >] as Mappable>::Value::default(), + tests::generate_key + ); + } + }; +} + +temporal_registry!(Address, Raw); +temporal_registry!(AssetId, Raw); +temporal_registry!(ContractId, Raw); +temporal_registry!(ScriptCode, ScriptCodeCodec); +temporal_registry!(PredicateCode, PredicateCodeCodec); + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(test)] + fuel_core_storage::basic_storage_tests!( + DaCompressionTemporalRegistryIndex, + ReverseKey::Address(Address::zeroed()), + RegistryKey::ZERO + ); + + #[cfg(test)] + fuel_core_storage::basic_storage_tests!( + DaCompressionTemporalRegistryTimestamps, + TimestampKey { + keyspace: timestamps::TimestampKeyspace::Address, + key: RegistryKey::ZERO + }, + Tai64::UNIX_EPOCH + ); + + #[cfg(test)] + fuel_core_storage::basic_storage_tests!( + DaCompressionTemporalRegistryEvictorCache, + MetadataKey::Address, + RegistryKey::ZERO + ); + + fuel_core_storage::basic_storage_tests!( + DaCompressedBlocks, + ::Key::default(), + ::Value::default() + ); + + #[allow(clippy::arithmetic_side_effects)] // Test code, and also safe + pub fn generate_key(rng: &mut impl rand::Rng) -> RegistryKey { + let raw_key: u32 = rng.gen_range(0..2u32.pow(24) - 2); + RegistryKey::try_from(raw_key).unwrap() + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/da_compression/evictor_cache.rs b/crates/fuel-core/src/graphql_api/storage/da_compression/evictor_cache.rs new file mode 100644 index 00000000000..870d02722f6 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/da_compression/evictor_cache.rs @@ -0,0 +1,34 @@ +/// The metadata key used by `DaCompressionTemporalRegistryEvictorCache` table to +/// store progress of the evictor. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + strum::EnumCount, +)] +pub enum MetadataKey { + Address, + AssetId, + ContractId, + ScriptCode, + PredicateCode, +} + +#[cfg(feature = "test-helpers")] +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> MetadataKey { + use strum::EnumCount; + match rng.next_u32() as usize % MetadataKey::COUNT { + 0 => MetadataKey::Address, + 1 => MetadataKey::AssetId, + 2 => MetadataKey::ContractId, + 3 => MetadataKey::ScriptCode, + 4 => MetadataKey::PredicateCode, + _ => unreachable!("New metadata key is added but not supported here"), + } + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/da_compression/predicate_code_codec.rs b/crates/fuel-core/src/graphql_api/storage/da_compression/predicate_code_codec.rs new file mode 100644 index 00000000000..6c165c09f3a --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/da_compression/predicate_code_codec.rs @@ -0,0 +1,28 @@ +use fuel_core_storage::codec::{ + Decode, + Encode, +}; +use fuel_core_types::fuel_tx::input::PredicateCode; +use std::{ + borrow::Cow, + ops::Deref, +}; + +// TODO: Remove this codec when the `PredicateCode` implements +// `AsRef<[u8]>` and `TryFrom<[u8]>` and use `Raw` codec instead. + +pub struct PredicateCodeCodec; + +impl Encode for PredicateCodeCodec { + type Encoder<'a> = Cow<'a, [u8]>; + + fn encode(t: &PredicateCode) -> Self::Encoder<'_> { + Cow::Borrowed(t.deref()) + } +} + +impl Decode for PredicateCodeCodec { + fn decode(bytes: &[u8]) -> anyhow::Result { + Ok(bytes.to_vec().into()) + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/da_compression/reverse_key.rs b/crates/fuel-core/src/graphql_api/storage/da_compression/reverse_key.rs new file mode 100644 index 00000000000..3c24d3a817f --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/da_compression/reverse_key.rs @@ -0,0 +1,82 @@ +use fuel_core_types::{ + fuel_tx::{ + input::PredicateCode, + ScriptCode, + }, + fuel_types::{ + Address, + AssetId, + Bytes32, + ContractId, + }, +}; +use std::ops::Deref; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + strum::EnumCount, +)] +/// The reverse key for the temporal registry index. +/// By this key we can find the registry key from the temporal registry. +pub enum ReverseKey { + Address(Address), + AssetId(AssetId), + ContractId(ContractId), + /// Hash of the script code. + ScriptCode(Bytes32), + /// Hash of the predicate code. + PredicateCode(Bytes32), +} + +impl From<&Address> for ReverseKey { + fn from(address: &Address) -> Self { + Self::Address(*address) + } +} + +impl From<&AssetId> for ReverseKey { + fn from(asset_id: &AssetId) -> Self { + Self::AssetId(*asset_id) + } +} + +impl From<&ContractId> for ReverseKey { + fn from(contract_id: &ContractId) -> Self { + Self::ContractId(*contract_id) + } +} + +impl From<&ScriptCode> for ReverseKey { + fn from(script_code: &ScriptCode) -> Self { + let hash = fuel_core_types::fuel_crypto::Hasher::hash(script_code.deref()); + ReverseKey::ScriptCode(hash) + } +} + +impl From<&PredicateCode> for ReverseKey { + fn from(predicate_code: &PredicateCode) -> Self { + let hash = fuel_core_types::fuel_crypto::Hasher::hash(predicate_code.deref()); + ReverseKey::PredicateCode(hash) + } +} + +#[cfg(feature = "test-helpers")] +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> ReverseKey { + use strum::EnumCount; + match rng.next_u32() as usize % ReverseKey::COUNT { + 0 => ReverseKey::Address(Address::default()), + 1 => ReverseKey::AssetId(AssetId::default()), + 2 => ReverseKey::ContractId(ContractId::default()), + 3 => ReverseKey::ScriptCode(Bytes32::default()), + 4 => ReverseKey::PredicateCode(Bytes32::default()), + _ => unreachable!("New reverse key is added but not supported here"), + } + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/da_compression/script_code_codec.rs b/crates/fuel-core/src/graphql_api/storage/da_compression/script_code_codec.rs new file mode 100644 index 00000000000..a4d6c8d1ac3 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/da_compression/script_code_codec.rs @@ -0,0 +1,28 @@ +use fuel_core_storage::codec::{ + Decode, + Encode, +}; +use fuel_core_types::fuel_tx::ScriptCode; +use std::{ + borrow::Cow, + ops::Deref, +}; + +// TODO: Remove this codec when the `ScriptCode` implements +// `AsRef<[u8]>` and `TryFrom<[u8]>` and use `Raw` codec instead. + +pub struct ScriptCodeCodec; + +impl Encode for ScriptCodeCodec { + type Encoder<'a> = Cow<'a, [u8]>; + + fn encode(t: &ScriptCode) -> Self::Encoder<'_> { + Cow::Borrowed(t.deref()) + } +} + +impl Decode for ScriptCodeCodec { + fn decode(bytes: &[u8]) -> anyhow::Result { + Ok(bytes.to_vec().into()) + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/da_compression/timestamps.rs b/crates/fuel-core/src/graphql_api/storage/da_compression/timestamps.rs new file mode 100644 index 00000000000..dc8f016f50b --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/da_compression/timestamps.rs @@ -0,0 +1,57 @@ +use fuel_core_types::fuel_compression::RegistryKey; + +/// The metadata key used by `DaCompressionTemporalRegistryTimsetamps` table to +/// keep track of when each key was last updated. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TimestampKey { + /// The column where the key is stored. + pub keyspace: TimestampKeyspace, + /// The key itself. + pub key: RegistryKey, +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + strum::EnumCount, +)] +pub enum TimestampKeyspace { + Address, + AssetId, + ContractId, + ScriptCode, + PredicateCode, +} + +#[cfg(feature = "test-helpers")] +impl rand::distributions::Distribution for rand::distributions::Standard { + #![allow(clippy::arithmetic_side_effects)] // Test-only code, and also safe + fn sample(&self, rng: &mut R) -> TimestampKey { + TimestampKey { + keyspace: rng.gen(), + key: RegistryKey::try_from(rng.gen_range(0..2u32.pow(24) - 2)).unwrap(), + } + } +} + +#[cfg(feature = "test-helpers")] +impl rand::distributions::Distribution + for rand::distributions::Standard +{ + fn sample(&self, rng: &mut R) -> TimestampKeyspace { + use strum::EnumCount; + match rng.next_u32() as usize % TimestampKeyspace::COUNT { + 0 => TimestampKeyspace::Address, + 1 => TimestampKeyspace::AssetId, + 2 => TimestampKeyspace::ContractId, + 3 => TimestampKeyspace::ScriptCode, + 4 => TimestampKeyspace::PredicateCode, + _ => unreachable!("New metadata key is added but not supported here"), + } + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/old.rs b/crates/fuel-core/src/graphql_api/storage/old.rs index a083b55c5f3..adb9c3b7f3d 100644 --- a/crates/fuel-core/src/graphql_api/storage/old.rs +++ b/crates/fuel-core/src/graphql_api/storage/old.rs @@ -15,13 +15,15 @@ use fuel_core_storage::{ structured_storage::TableWithBlueprint, Mappable, }; -use fuel_core_txpool::types::TxId; use fuel_core_types::{ blockchain::{ block::CompressedBlock, consensus::Consensus, }, - fuel_tx::Transaction, + fuel_tx::{ + Transaction, + TxId, + }, fuel_types::BlockHeight, }; diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 1c19788d194..959733d4919 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -1,12 +1,17 @@ -use super::storage::old::{ - OldFuelBlockConsensus, - OldFuelBlocks, - OldTransactions, +use super::{ + da_compression::da_compress_block, + storage::old::{ + OldFuelBlockConsensus, + OldFuelBlocks, + OldTransactions, + }, }; use crate::{ fuel_core_graphql_api::{ - ports, - ports::worker::OffChainDatabaseTransaction, + ports::{ + self, + worker::OffChainDatabaseTransaction, + }, storage::{ blocks::FuelBlockIdsToHeights, coins::{ @@ -37,7 +42,6 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsMut, }; -use fuel_core_txpool::types::TxId; use fuel_core_types::{ blockchain::{ block::{ @@ -62,6 +66,7 @@ use fuel_core_types::{ Input, Output, Transaction, + TxId, UniqueIdentifier, }, fuel_types::{ @@ -93,9 +98,16 @@ use std::{ #[cfg(test)] mod tests; +#[derive(Debug, Clone)] +pub enum DaCompressionConfig { + Disabled, + Enabled(fuel_core_compression::config::Config), +} + /// The initialization task recovers the state of the GraphQL service database on startup. pub struct InitializeTask { chain_id: ChainId, + da_compression_config: DaCompressionConfig, continue_on_error: bool, tx_pool: TxPool, blocks_events: BoxStream, @@ -111,6 +123,7 @@ pub struct Task { block_importer: BoxStream, database: D, chain_id: ChainId, + da_compression_config: DaCompressionConfig, continue_on_error: bool, } @@ -134,7 +147,7 @@ where let height = block.header().height(); let block_id = block.id(); transaction - .storage::() + .storage_as_mut::() .insert(&block_id, height)?; let total_tx_count = transaction @@ -146,6 +159,13 @@ where &mut transaction, )?; + match self.da_compression_config { + DaCompressionConfig::Disabled => {} + DaCompressionConfig::Enabled(config) => { + da_compress_block(config, block, &result.events, &mut transaction)?; + } + } + transaction.commit()?; for status in result.tx_status.iter() { @@ -455,6 +475,7 @@ where let InitializeTask { chain_id, + da_compression_config, tx_pool, block_importer, blocks_events, @@ -468,6 +489,7 @@ where block_importer: blocks_events, database: off_chain_database, chain_id, + da_compression_config, continue_on_error, }; @@ -579,6 +601,7 @@ pub fn new_service( on_chain_database: OnChain, off_chain_database: OffChain, chain_id: ChainId, + da_compression_config: DaCompressionConfig, continue_on_error: bool, ) -> ServiceRunner> where @@ -594,6 +617,7 @@ where on_chain_database, off_chain_database, chain_id, + da_compression_config, continue_on_error, }) } diff --git a/crates/fuel-core/src/graphql_api/worker_service/tests.rs b/crates/fuel-core/src/graphql_api/worker_service/tests.rs index b6eef2c7826..8b9ad758975 100644 --- a/crates/fuel-core/src/graphql_api/worker_service/tests.rs +++ b/crates/fuel-core/src/graphql_api/worker_service/tests.rs @@ -81,6 +81,7 @@ fn worker_task_with_block_importer_and_db( block_importer, database, chain_id, + da_compression_config: DaCompressionConfig::Disabled, continue_on_error: false, } } diff --git a/crates/fuel-core/src/p2p_test_helpers.rs b/crates/fuel-core/src/p2p_test_helpers.rs index f72b4c348b0..fbc9f32afb4 100644 --- a/crates/fuel-core/src/p2p_test_helpers.rs +++ b/crates/fuel-core/src/p2p_test_helpers.rs @@ -6,8 +6,13 @@ use crate::{ CoinConfigGenerator, }, combined_database::CombinedDatabase, - database::Database, + database::{ + database_description::off_chain::OffChain, + Database, + }, + fuel_core_graphql_api::storage::transactions::TransactionStatuses, p2p::Multiaddr, + schema::tx::types::TransactionStatus, service::{ Config, FuelService, @@ -33,7 +38,6 @@ use fuel_core_poa::{ Trigger, }; use fuel_core_storage::{ - tables::Transactions, transactional::AtomicView, StorageAsRef, }; @@ -59,7 +63,6 @@ use fuel_core_types::{ services::p2p::GossipsubMessageAcceptance, }; use futures::StreamExt; -use itertools::Itertools; use rand::{ rngs::StdRng, SeedableRng, @@ -70,7 +73,6 @@ use std::{ Index, IndexMut, }, - sync::Arc, time::Duration, }; use tokio::sync::broadcast; @@ -491,24 +493,34 @@ impl Node { /// Wait for the node to reach consistency with the given transactions. pub async fn consistency(&mut self, txs: &HashMap) { - let Self { db, .. } = self; - let mut blocks = self.node.shared.block_importer.block_stream(); - while !not_found_txs(db, txs).is_empty() { - tokio::select! { - result = blocks.next() => { - result.unwrap(); - } - _ = self.node.await_shutdown() => { - panic!("Got a stop signal") + let db = self.node.shared.database.off_chain(); + loop { + let not_found = not_found_txs(db, txs); + + if not_found.is_empty() { + break; + } + + let tx_id = not_found[0]; + let mut wait_transaction = + self.node.transaction_status_change(tx_id).await.unwrap(); + + loop { + tokio::select! { + result = wait_transaction.next() => { + let status = result.unwrap().unwrap(); + + if matches!(status, TransactionStatus::Failed { .. }) + || matches!(status, TransactionStatus::Success { .. }) { + break + } + } + _ = self.node.await_shutdown() => { + panic!("Got a stop signal") + } } } } - - let count = db - .all_transactions(None, None) - .filter_ok(|tx| tx.is_script()) - .count(); - assert_eq!(count, txs.len()); } /// Wait for the node to reach consistency with the given transactions within 10 seconds. @@ -533,20 +545,15 @@ impl Node { pub async fn insert_txs(&self) -> HashMap { let mut expected = HashMap::new(); for tx in &self.test_txs { - let tx_result = self - .node + let tx_id = tx.id(&ChainId::default()); + self.node .shared .txpool_shared_state - .insert(vec![Arc::new(tx.clone())]) + .insert(tx.clone()) .await - .pop() - .unwrap() .unwrap(); - let tx = Transaction::from(tx_result.inserted.as_ref()); - expected.insert(tx.id(&ChainId::default()), tx); - - assert!(tx_result.removed.is_empty()); + expected.insert(tx_id, tx.clone()); } expected } @@ -570,13 +577,17 @@ impl Node { } fn not_found_txs<'iter>( - db: &'iter Database, + db: &'iter Database, txs: &'iter HashMap, ) -> Vec { let mut not_found = vec![]; txs.iter().for_each(|(id, tx)| { assert_eq!(id, &tx.id(&Default::default())); - if !db.storage::().contains_key(id).unwrap() { + let found = db + .storage::() + .contains_key(id) + .unwrap(); + if !found { not_found.push(*id); } }); diff --git a/crates/fuel-core/src/query.rs b/crates/fuel-core/src/query.rs index c5d3d0f6988..fc2dc79ea9b 100644 --- a/crates/fuel-core/src/query.rs +++ b/crates/fuel-core/src/query.rs @@ -9,6 +9,8 @@ mod subscriptions; mod tx; mod upgrades; +pub mod da_compressed; + // TODO: Remove reexporting of everything pub use balance::*; pub use blob::*; diff --git a/crates/fuel-core/src/query/da_compressed.rs b/crates/fuel-core/src/query/da_compressed.rs new file mode 100644 index 00000000000..669e55d584e --- /dev/null +++ b/crates/fuel-core/src/query/da_compressed.rs @@ -0,0 +1,16 @@ +use crate::graphql_api::ports::DatabaseDaCompressedBlocks; +use fuel_core_storage::Result as StorageResult; +use fuel_core_types::fuel_types::BlockHeight; + +pub trait DaCompressedBlockData: Send + Sync { + fn da_compressed_block(&self, id: &BlockHeight) -> StorageResult>; +} + +impl DaCompressedBlockData for D +where + D: DatabaseDaCompressedBlocks + ?Sized + Send + Sync, +{ + fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult> { + self.da_compressed_block(height) + } +} diff --git a/crates/fuel-core/src/query/message/test.rs b/crates/fuel-core/src/query/message/test.rs index 258de7fc7be..3078f3a7de9 100644 --- a/crates/fuel-core/src/query/message/test.rs +++ b/crates/fuel-core/src/query/message/test.rs @@ -1,6 +1,5 @@ use std::ops::Deref; -use fuel_core_txpool::types::ContractId; use fuel_core_types::{ blockchain::header::{ ApplicationHeader, @@ -10,6 +9,7 @@ use fuel_core_types::{ entities::relayer::message::MerkleProof, fuel_tx::{ AssetId, + ContractId, Script, Transaction, }, diff --git a/crates/fuel-core/src/query/subscriptions.rs b/crates/fuel-core/src/query/subscriptions.rs index 701632fd16c..120bd73692c 100644 --- a/crates/fuel-core/src/query/subscriptions.rs +++ b/crates/fuel-core/src/query/subscriptions.rs @@ -1,6 +1,6 @@ use crate::schema::tx::types::TransactionStatus as ApiTxStatus; use fuel_core_storage::Result as StorageResult; -use fuel_core_txpool::service::TxStatusMessage; +use fuel_core_txpool::TxStatusMessage; use fuel_core_types::{ fuel_types::Bytes32, services::txpool::TransactionStatus as TxPoolTxStatus, @@ -17,20 +17,11 @@ mod test; #[cfg_attr(test, mockall::automock)] pub(crate) trait TxnStatusChangeState { /// Return the transaction status from the tx pool and database. - fn get_tx_status(&self, id: Bytes32) -> StorageResult>; -} - -impl TxnStatusChangeState for F -where - F: Fn(Bytes32) -> StorageResult> + Send + Sync, -{ - fn get_tx_status(&self, id: Bytes32) -> StorageResult> { - self(id) - } + async fn get_tx_status(&self, id: Bytes32) -> StorageResult>; } #[tracing::instrument(skip(state, stream), fields(transaction_id = %transaction_id))] -pub(crate) fn transaction_status_change<'a, State>( +pub(crate) async fn transaction_status_change<'a, State>( state: State, stream: BoxStream<'a, TxStatusMessage>, transaction_id: Bytes32, @@ -42,6 +33,7 @@ where // has a status. let check_db_first = state .get_tx_status(transaction_id) + .await .transpose() .map(TxStatusMessage::from); diff --git a/crates/fuel-core/src/query/subscriptions/test.rs b/crates/fuel-core/src/query/subscriptions/test.rs index c20b668a459..67e01848b7c 100644 --- a/crates/fuel-core/src/query/subscriptions/test.rs +++ b/crates/fuel-core/src/query/subscriptions/test.rs @@ -19,7 +19,10 @@ //! - `tx_status_message()`: Generates a TxStatusMessage //! - `transaction_status()`: Generates a TransactionStatus //! - `input_stream()`: Generates a Vec of length 0 to 5 -use fuel_core_txpool::service::TxStatusMessage; +use fuel_core_txpool::{ + error::RemovedReason, + TxStatusMessage, +}; use fuel_core_types::{ fuel_types::Bytes32, services::txpool::TransactionStatus, @@ -76,7 +79,7 @@ fn failed() -> TransactionStatus { /// Returns a TransactionStatus with SqueezedOut status and an empty error message fn squeezed() -> TransactionStatus { TransactionStatus::SqueezedOut { - reason: fuel_core_txpool::Error::SqueezedOut(String::new()).to_string(), + reason: fuel_core_txpool::error::Error::Removed(RemovedReason::Ttl).to_string(), } } @@ -251,6 +254,7 @@ fn test_tsc_inner( let stream = futures::stream::iter(stream).boxed(); super::transaction_status_change(mock_state, stream, txn_id(0)) + .await .collect::>() .await }) diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index 956701ccd99..1d2f1531363 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -14,11 +14,11 @@ use fuel_core_storage::{ tables::Transactions, Result as StorageResult, }; -use fuel_core_txpool::types::TxId; use fuel_core_types::{ fuel_tx::{ Receipt, Transaction, + TxId, TxPointer, }, fuel_types::Address, diff --git a/crates/fuel-core/src/schema.rs b/crates/fuel-core/src/schema.rs index 747f2740151..bd9e550d448 100644 --- a/crates/fuel-core/src/schema.rs +++ b/crates/fuel-core/src/schema.rs @@ -32,6 +32,7 @@ pub mod block; pub mod chain; pub mod coins; pub mod contract; +pub mod da_compressed; pub mod dap; pub mod health; pub mod message; @@ -54,6 +55,7 @@ pub struct Query( tx::TxQuery, health::HealthQuery, coins::CoinQuery, + da_compressed::DaCompressedBlockQuery, contract::ContractQuery, contract::ContractBalanceQuery, node_info::NodeQuery, diff --git a/crates/fuel-core/src/schema/blob.rs b/crates/fuel-core/src/schema/blob.rs index ae043e81049..938c2a6a1f3 100644 --- a/crates/fuel-core/src/schema/blob.rs +++ b/crates/fuel-core/src/schema/blob.rs @@ -49,6 +49,7 @@ pub struct BlobQuery; #[Object] impl BlobQuery { + #[graphql(complexity = "QUERY_COSTS.storage_read + child_complexity")] async fn blob( &self, ctx: &Context<'_>, diff --git a/crates/fuel-core/src/schema/block.rs b/crates/fuel-core/src/schema/block.rs index ee9c3e78f8c..81a9bf3c0af 100644 --- a/crates/fuel-core/src/schema/block.rs +++ b/crates/fuel-core/src/schema/block.rs @@ -125,6 +125,7 @@ impl Block { Ok(query.consensus(height)?.try_into()?) } + #[graphql(complexity = "QUERY_COSTS.block_transactions_ids")] async fn transaction_ids(&self) -> Vec { self.0 .transactions() @@ -134,8 +135,7 @@ impl Block { } // Assume that in average we have 32 transactions per block. - #[graphql(complexity = "QUERY_COSTS.storage_iterator\ - + (QUERY_COSTS.storage_read + child_complexity) * 32")] + #[graphql(complexity = "QUERY_COSTS.block_transactions + child_complexity")] async fn transactions( &self, ctx: &Context<'_>, @@ -246,7 +246,7 @@ pub struct BlockQuery; #[Object] impl BlockQuery { - #[graphql(complexity = "2 * QUERY_COSTS.storage_read + child_complexity")] + #[graphql(complexity = "QUERY_COSTS.block_header + child_complexity")] async fn block( &self, ctx: &Context<'_>, @@ -276,9 +276,8 @@ impl BlockQuery { } #[graphql(complexity = "{\ - QUERY_COSTS.storage_iterator\ - + (QUERY_COSTS.storage_read + first.unwrap_or_default() as usize) * child_complexity \ - + (QUERY_COSTS.storage_read + last.unwrap_or_default() as usize) * child_complexity\ + (QUERY_COSTS.block_header + child_complexity) \ + * (first.unwrap_or_default() as usize + last.unwrap_or_default() as usize) \ }")] async fn blocks( &self, @@ -305,7 +304,7 @@ pub struct HeaderQuery; #[Object] impl HeaderQuery { - #[graphql(complexity = "QUERY_COSTS.storage_read + child_complexity")] + #[graphql(complexity = "QUERY_COSTS.block_header + child_complexity")] async fn header( &self, ctx: &Context<'_>, @@ -319,9 +318,8 @@ impl HeaderQuery { } #[graphql(complexity = "{\ - QUERY_COSTS.storage_iterator\ - + (QUERY_COSTS.storage_read + first.unwrap_or_default() as usize) * child_complexity \ - + (QUERY_COSTS.storage_read + last.unwrap_or_default() as usize) * child_complexity\ + (QUERY_COSTS.block_header + child_complexity) \ + * (first.unwrap_or_default() as usize + last.unwrap_or_default() as usize) \ }")] async fn headers( &self, @@ -374,13 +372,14 @@ impl BlockMutation { start_timestamp: Option, blocks_to_produce: U32, ) -> async_graphql::Result { - let consensus_module = ctx.data_unchecked::(); let config = ctx.data_unchecked::().clone(); if !config.debug { return Err(anyhow!("`debug` must be enabled to use this endpoint").into()) } + let consensus_module = ctx.data_unchecked::(); + let start_time = start_timestamp.map(|timestamp| timestamp.0); let blocks_to_produce: u32 = blocks_to_produce.into(); consensus_module diff --git a/crates/fuel-core/src/schema/da_compressed.rs b/crates/fuel-core/src/schema/da_compressed.rs new file mode 100644 index 00000000000..3af336f8ba9 --- /dev/null +++ b/crates/fuel-core/src/schema/da_compressed.rs @@ -0,0 +1,51 @@ +use super::{ + scalars::HexString, + ReadViewProvider, +}; +use crate::{ + fuel_core_graphql_api::{ + IntoApiResult, + QUERY_COSTS, + }, + query::da_compressed::DaCompressedBlockData, + schema::scalars::U32, +}; +use async_graphql::{ + Context, + Object, +}; + +pub struct DaCompressedBlock { + bytes: Vec, +} + +impl From> for DaCompressedBlock { + fn from(bytes: Vec) -> Self { + Self { bytes } + } +} + +#[Object] +impl DaCompressedBlock { + async fn bytes(&self) -> HexString { + HexString(self.bytes.clone()) + } +} + +#[derive(Default)] +pub struct DaCompressedBlockQuery; + +#[Object] +impl DaCompressedBlockQuery { + #[graphql(complexity = "QUERY_COSTS.da_compressed_block_read")] + async fn da_compressed_block( + &self, + ctx: &Context<'_>, + #[graphql(desc = "Height of the block")] height: U32, + ) -> async_graphql::Result> { + let query = ctx.read_view()?; + query + .da_compressed_block(&height.0.into()) + .into_api_result() + } +} diff --git a/crates/fuel-core/src/schema/gas_price.rs b/crates/fuel-core/src/schema/gas_price.rs index 577d68cec9e..f1ded5fc106 100644 --- a/crates/fuel-core/src/schema/gas_price.rs +++ b/crates/fuel-core/src/schema/gas_price.rs @@ -46,7 +46,7 @@ pub struct LatestGasPriceQuery {} #[Object] impl LatestGasPriceQuery { - #[graphql(complexity = "2 * QUERY_COSTS.storage_read")] + #[graphql(complexity = "QUERY_COSTS.block_header")] async fn latest_gas_price( &self, ctx: &Context<'_>, diff --git a/crates/fuel-core/src/schema/node_info.rs b/crates/fuel-core/src/schema/node_info.rs index ef4a5965f72..f58e28a1d3b 100644 --- a/crates/fuel-core/src/schema/node_info.rs +++ b/crates/fuel-core/src/schema/node_info.rs @@ -76,7 +76,7 @@ impl NodeQuery { utxo_validation: config.utxo_validation, vm_backtrace: config.vm_backtrace, max_tx: (config.max_tx as u64).into(), - max_depth: (config.max_txpool_depth as u64).into(), + max_depth: (config.max_txpool_dependency_chain_length as u64).into(), node_version: VERSION.to_owned(), }) } diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index db77716d722..d0a1474340a 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -1,3 +1,4 @@ +use super::scalars::U64; use crate::{ fuel_core_graphql_api::{ api_service::{ @@ -9,11 +10,16 @@ use crate::{ IntoApiResult, QUERY_COSTS, }, + graphql_api::{ + database::ReadView, + ports::MemoryPool, + }, query::{ transaction_status_change, BlockQueryData, SimpleTransactionData, TransactionQueryData, + TxnStatusChangeState, }, schema::{ scalars::{ @@ -43,12 +49,10 @@ use fuel_core_storage::{ Error as StorageError, Result as StorageResult, }; -use fuel_core_txpool::{ - ports::MemoryPool, - service::TxStatusMessage, -}; +use fuel_core_txpool::TxStatusMessage; use fuel_core_types::{ fuel_tx::{ + Bytes32, Cacheable, Transaction as FuelTx, UniqueIdentifier, @@ -69,8 +73,8 @@ use futures::{ }; use itertools::Itertools; use std::{ + borrow::Cow, iter, - sync::Arc, }; use tokio_stream::StreamExt; use types::{ @@ -78,8 +82,6 @@ use types::{ Transaction, }; -use super::scalars::U64; - pub mod input; pub mod output; pub mod receipt; @@ -101,7 +103,7 @@ impl TxQuery { let id = id.0; let txpool = ctx.data_unchecked::(); - if let Some(transaction) = txpool.transaction(id) { + if let Some(transaction) = txpool.transaction(id).await? { Ok(Some(Transaction(transaction, id))) } else { query @@ -111,10 +113,10 @@ impl TxQuery { } } + // We assume that each block has 100 transactions. #[graphql(complexity = "{\ - QUERY_COSTS.storage_iterator\ - + (QUERY_COSTS.storage_read + first.unwrap_or_default() as usize) * child_complexity \ - + (QUERY_COSTS.storage_read + last.unwrap_or_default() as usize) * child_complexity\ + (QUERY_COSTS.tx_get + child_complexity) \ + * (first.unwrap_or_default() as usize + last.unwrap_or_default() as usize) }")] async fn transactions( &self, @@ -227,6 +229,8 @@ impl TxQuery { ctx: &Context<'_>, tx: HexString, ) -> async_graphql::Result { + let query = ctx.read_view()?.into_owned(); + let mut tx = FuelTx::from_bytes(&tx.0)?; let params = ctx @@ -238,7 +242,7 @@ impl TxQuery { let parameters = CheckPredicateParams::from(params.as_ref()); let tx = tokio_rayon::spawn_fifo(move || { - let result = tx.estimate_predicates(¶meters, memory); + let result = tx.estimate_predicates(¶meters, memory, &query); result.map(|_| tx) }) .await @@ -328,11 +332,10 @@ impl TxMutation { .latest_consensus_params(); let tx = FuelTx::from_bytes(&tx.0)?; - let _: Vec<_> = txpool - .insert(vec![Arc::new(tx.clone())]) + txpool + .insert(tx.clone()) .await - .into_iter() - .try_collect()?; + .map_err(|e| anyhow::anyhow!(e))?; let id = tx.id(¶ms.chain_id()); let tx = Transaction(tx, id); @@ -368,18 +371,12 @@ impl TxStatusSubscription { let rx = txpool.tx_update_subscribe(id.into())?; let query = ctx.read_view()?; - Ok(transaction_status_change( - move |id| match query.tx_status(&id) { - Ok(status) => Ok(Some(status)), - Err(StorageError::NotFound(_, _)) => Ok(txpool - .submission_time(id) - .map(|time| txpool::TransactionStatus::Submitted { time })), - Err(err) => Err(err), - }, - rx, - id.into(), + let status_change_state = StatusChangeState { txpool, query }; + Ok( + transaction_status_change(status_change_state, rx, id.into()) + .await + .map_err(async_graphql::Error::from), ) - .map_err(async_graphql::Error::from)) } /// Submits transaction to the `TxPool` and await either confirmation or failure. @@ -427,11 +424,7 @@ async fn submit_and_await_status<'a>( let tx_id = tx.id(¶ms.chain_id()); let subscription = txpool.tx_update_subscribe(tx_id)?; - let _: Vec<_> = txpool - .insert(vec![Arc::new(tx)]) - .await - .into_iter() - .try_collect()?; + txpool.insert(tx).await?; Ok(subscription .map(move |event| match event { @@ -445,3 +438,26 @@ async fn submit_and_await_status<'a>( }) .take(2)) } + +struct StatusChangeState<'a> { + query: Cow<'a, ReadView>, + txpool: &'a TxPool, +} + +impl<'a> TxnStatusChangeState for StatusChangeState<'a> { + async fn get_tx_status( + &self, + id: Bytes32, + ) -> StorageResult> { + match self.query.tx_status(&id) { + Ok(status) => Ok(Some(status)), + Err(StorageError::NotFound(_, _)) => Ok(self + .txpool + .submission_time(id) + .await + .map_err(|e| anyhow::anyhow!(e))? + .map(|time| txpool::TransactionStatus::Submitted { time })), + Err(err) => Err(err), + } + } +} diff --git a/crates/fuel-core/src/schema/tx/types.rs b/crates/fuel-core/src/schema/tx/types.rs index 9496a5bda22..820271430e2 100644 --- a/crates/fuel-core/src/schema/tx/types.rs +++ b/crates/fuel-core/src/schema/tx/types.rs @@ -182,7 +182,7 @@ impl SuccessStatus { self.block_height.into() } - #[graphql(complexity = "QUERY_COSTS.storage_read + child_complexity")] + #[graphql(complexity = "QUERY_COSTS.block_header + child_complexity")] async fn block(&self, ctx: &Context<'_>) -> async_graphql::Result { let query = ctx.read_view()?; let block = query.block(&self.block_height)?; @@ -238,7 +238,7 @@ impl FailureStatus { self.block_height.into() } - #[graphql(complexity = "QUERY_COSTS.storage_read + child_complexity")] + #[graphql(complexity = "QUERY_COSTS.block_header + child_complexity")] async fn block(&self, ctx: &Context<'_>) -> async_graphql::Result { let query = ctx.read_view()?; let block = query.block(&self.block_height)?; @@ -693,7 +693,7 @@ impl Transaction { } } - #[graphql(complexity = "QUERY_COSTS.storage_read + child_complexity")] + #[graphql(complexity = "QUERY_COSTS.tx_status_read + child_complexity")] async fn status( &self, ctx: &Context<'_>, @@ -701,7 +701,9 @@ impl Transaction { let id = self.1; let query = ctx.read_view()?; let txpool = ctx.data_unchecked::(); - get_tx_status(id, query.as_ref(), txpool).map_err(Into::into) + get_tx_status(id, query.as_ref(), txpool) + .await + .map_err(Into::into) } async fn script(&self) -> Option { @@ -846,7 +848,7 @@ impl Transaction { } } - #[graphql(complexity = "QUERY_COSTS.raw_payload")] + #[graphql(complexity = "QUERY_COSTS.tx_raw_payload")] /// Return the transaction bytes using canonical encoding async fn raw_payload(&self) -> HexString { HexString(self.0.clone().to_bytes()) @@ -984,7 +986,7 @@ impl DryRunTransactionExecutionStatus { } #[tracing::instrument(level = "debug", skip(query, txpool), ret, err)] -pub(crate) fn get_tx_status( +pub(crate) async fn get_tx_status( id: fuel_core_types::fuel_types::Bytes32, query: &ReadView, txpool: &TxPool, @@ -997,12 +999,18 @@ pub(crate) fn get_tx_status( let status = TransactionStatus::new(id, status); Ok(Some(status)) } - None => match txpool.submission_time(id) { - Some(submitted_time) => Ok(Some(TransactionStatus::Submitted( - SubmittedStatus(submitted_time), - ))), - _ => Ok(None), - }, + None => { + let submitted_time = txpool + .submission_time(id) + .await + .map_err(|e| StorageError::Other(anyhow::anyhow!(e)))?; + match submitted_time { + Some(submitted_time) => Ok(Some(TransactionStatus::Submitted( + SubmittedStatus(submitted_time), + ))), + _ => Ok(None), + } + } } } diff --git a/crates/fuel-core/src/schema/upgrades.rs b/crates/fuel-core/src/schema/upgrades.rs index bda98240304..d9e62906a1e 100644 --- a/crates/fuel-core/src/schema/upgrades.rs +++ b/crates/fuel-core/src/schema/upgrades.rs @@ -74,6 +74,7 @@ impl StateTransitionBytecode { HexString(self.root.to_vec()) } + #[graphql(complexity = "QUERY_COSTS.state_transition_bytecode_read")] async fn bytecode( &self, ctx: &Context<'_>, diff --git a/crates/fuel-core/src/service/adapters.rs b/crates/fuel-core/src/service/adapters.rs index 7c356611967..f55d4572335 100644 --- a/crates/fuel-core/src/service/adapters.rs +++ b/crates/fuel-core/src/service/adapters.rs @@ -10,6 +10,7 @@ use fuel_core_poa::{ }; use fuel_core_services::stream::BoxStream; use fuel_core_storage::transactional::Changes; +use fuel_core_txpool::BorrowedTxPool; #[cfg(feature = "p2p")] use fuel_core_types::services::p2p::peer_reputation::AppScore; use fuel_core_types::{ @@ -18,7 +19,6 @@ use fuel_core_types::{ consensus::Consensus, }, fuel_tx::Transaction, - fuel_types::BlockHeight, services::{ block_importer::SharedImportResult, block_producer::Components, @@ -101,17 +101,16 @@ impl TxPoolAdapter { } } -#[derive(Clone)] pub struct TransactionsSource { - txpool: TxPoolSharedState, - _block_height: BlockHeight, + tx_pool: BorrowedTxPool, + minimum_gas_price: u64, } impl TransactionsSource { - pub fn new(txpool: TxPoolSharedState, block_height: BlockHeight) -> Self { + pub fn new(minimum_gas_price: u64, tx_pool: BorrowedTxPool) -> Self { Self { - txpool, - _block_height: block_height, + tx_pool, + minimum_gas_price, } } } diff --git a/crates/fuel-core/src/service/adapters/consensus_module/poa.rs b/crates/fuel-core/src/service/adapters/consensus_module/poa.rs index adcd72d2d61..f3ebf270f91 100644 --- a/crates/fuel-core/src/service/adapters/consensus_module/poa.rs +++ b/crates/fuel-core/src/service/adapters/consensus_module/poa.rs @@ -26,18 +26,14 @@ use fuel_core_services::stream::BoxStream; use fuel_core_storage::transactional::Changes; use fuel_core_types::{ blockchain::block::Block, - fuel_tx::TxId, + fuel_tx::Bytes32, fuel_types::BlockHeight, services::{ block_importer::{ BlockImportInfo, UncommittedResult as UncommittedImporterResult, }, - executor::{ - Error as ExecutorError, - UncommittedResult, - }, - txpool::ArcPoolTx, + executor::UncommittedResult, }, tai64::Tai64, }; @@ -45,6 +41,7 @@ use std::path::{ Path, PathBuf, }; +use tokio::sync::watch; use tokio_stream::{ wrappers::BroadcastStream, StreamExt, @@ -81,27 +78,12 @@ impl ConsensusModulePort for PoAAdapter { } impl TransactionPool for TxPoolAdapter { - fn pending_number(&self) -> usize { - self.service.pending_number() - } - - fn total_consumable_gas(&self) -> u64 { - self.service.total_consumable_gas() - } - - fn remove_txs(&self, ids: Vec<(TxId, ExecutorError)>) -> Vec { - self.service.remove_txs( - ids.into_iter() - .map(|(tx_id, err)| (tx_id, err.to_string())) - .collect(), - ) + fn new_txs_watcher(&self) -> watch::Receiver<()> { + self.service.get_new_txs_notifier() } - fn transaction_status_events(&self) -> BoxStream { - Box::pin( - BroadcastStream::new(self.service.new_tx_notification_subscribe()) - .filter_map(|result| result.ok()), - ) + fn notify_skipped_txs(&self, tx_ids_and_reasons: Vec<(Bytes32, String)>) { + self.service.notify_skipped_txs(tx_ids_and_reasons) } } diff --git a/crates/fuel-core/src/service/adapters/executor.rs b/crates/fuel-core/src/service/adapters/executor.rs index 05d4400a1f4..663c219b001 100644 --- a/crates/fuel-core/src/service/adapters/executor.rs +++ b/crates/fuel-core/src/service/adapters/executor.rs @@ -3,10 +3,12 @@ use crate::{ service::adapters::TransactionsSource, }; use fuel_core_executor::ports::MaybeCheckedTransaction; +use fuel_core_txpool::Constraints; use fuel_core_types::{ blockchain::primitives::DaBlockHeight, services::relayer::Event, }; +use std::sync::Arc; impl fuel_core_executor::ports::TransactionsSource for TransactionsSource { fn next( @@ -15,18 +17,19 @@ impl fuel_core_executor::ports::TransactionsSource for TransactionsSource { transactions_limit: u16, block_transaction_size_limit: u32, ) -> Vec { - self.txpool - .select_transactions( - gas_limit, - transactions_limit, - block_transaction_size_limit, - ) + self.tx_pool + .exclusive_lock() + .extract_transactions_for_block(Constraints { + minimal_gas_price: self.minimum_gas_price, + max_gas: gas_limit, + maximum_txs: transactions_limit, + maximum_block_size: block_transaction_size_limit, + }) .into_iter() .map(|tx| { - MaybeCheckedTransaction::CheckedTransaction( - tx.as_ref().into(), - tx.used_consensus_parameters_version(), - ) + let transaction = Arc::unwrap_or_clone(tx); + let version = transaction.used_consensus_parameters_version(); + MaybeCheckedTransaction::CheckedTransaction(transaction.into(), version) }) .collect() } diff --git a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs index c1d79e3d1f3..aeb17627ccf 100644 --- a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs +++ b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs @@ -5,10 +5,7 @@ use fuel_core_gas_price_service::common::gas_price_algorithm::{ }; use fuel_core_producer::block_producer::gas_price::GasPriceProvider as ProducerGasPriceProvider; -use fuel_core_txpool::{ - ports::GasPriceProvider as TxPoolGasPriceProvider, - Result as TxPoolResult, -}; +use fuel_core_txpool::ports::GasPriceProvider as TxPoolGasPriceProvider; use fuel_core_types::fuel_types::BlockHeight; pub type Result = std::result::Result; @@ -54,8 +51,8 @@ impl FuelGasPriceProvider where A: GasPriceAlgorithm + Send + Sync, { - async fn next_gas_price(&self) -> u64 { - self.algorithm.next_gas_price().await + fn next_gas_price(&self) -> u64 { + self.algorithm.next_gas_price() } } @@ -65,17 +62,16 @@ where A: GasPriceAlgorithm + Send + Sync, { async fn next_gas_price(&self) -> anyhow::Result { - Ok(self.next_gas_price().await) + Ok(self.next_gas_price()) } } -#[async_trait::async_trait] impl TxPoolGasPriceProvider for FuelGasPriceProvider where - A: GasPriceAlgorithm + Send + Sync, + A: GasPriceAlgorithm + Send + Sync + 'static, { - async fn next_gas_price(&self) -> TxPoolResult { - Ok(self.next_gas_price().await) + fn next_gas_price(&self) -> u64 { + self.next_gas_price() } } diff --git a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/producer_gas_price_tests.rs b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/producer_gas_price_tests.rs index 81429785248..d8929cb4fdf 100644 --- a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/producer_gas_price_tests.rs +++ b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/producer_gas_price_tests.rs @@ -13,7 +13,7 @@ async fn gas_price__if_requested_block_height_is_latest_return_gas_price() { // when let expected_price = algo.next_gas_price(); - let actual_price = gas_price_provider.next_gas_price().await; + let actual_price = gas_price_provider.next_gas_price(); // then assert_eq!(expected_price, actual_price); diff --git a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/tx_pool_gas_price_tests.rs b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/tx_pool_gas_price_tests.rs index 81429785248..d8929cb4fdf 100644 --- a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/tx_pool_gas_price_tests.rs +++ b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider/tests/tx_pool_gas_price_tests.rs @@ -13,7 +13,7 @@ async fn gas_price__if_requested_block_height_is_latest_return_gas_price() { // when let expected_price = algo.next_gas_price(); - let actual_price = gas_price_provider.next_gas_price().await; + let actual_price = gas_price_provider.next_gas_price(); // then assert_eq!(expected_price, actual_price); diff --git a/crates/fuel-core/src/service/adapters/graphql_api.rs b/crates/fuel-core/src/service/adapters/graphql_api.rs index 5c1ecd9c80b..ff96e484ed9 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api.rs @@ -2,6 +2,7 @@ use super::{ BlockImporterAdapter, BlockProducerAdapter, ConsensusParametersProvider, + SharedMemoryPool, StaticGasPrice, }; use crate::{ @@ -15,19 +16,20 @@ use crate::{ P2pPort, TxPoolPort, }, - service::adapters::{ - import_result_provider::ImportResultProvider, - P2PAdapter, - TxPoolAdapter, + graphql_api::ports::MemoryPool, + service::{ + adapters::{ + import_result_provider::ImportResultProvider, + P2PAdapter, + TxPoolAdapter, + }, + vm_pool::MemoryFromPool, }, }; use async_trait::async_trait; use fuel_core_services::stream::BoxStream; use fuel_core_storage::Result as StorageResult; -use fuel_core_txpool::{ - service::TxStatusMessage, - types::TxId, -}; +use fuel_core_txpool::TxStatusMessage; use fuel_core_types::{ blockchain::header::ConsensusParametersVersion, entities::relayer::message::MerkleProof, @@ -35,16 +37,14 @@ use fuel_core_types::{ Bytes32, ConsensusParameters, Transaction, + TxId, }, fuel_types::BlockHeight, services::{ block_importer::SharedImportResult, executor::TransactionExecutionStatus, p2p::PeerInfo, - txpool::{ - InsertionResult, - TransactionStatus, - }, + txpool::TransactionStatus, }, tai64::Tai64, }; @@ -58,28 +58,36 @@ mod on_chain; #[async_trait] impl TxPoolPort for TxPoolAdapter { - fn transaction(&self, id: TxId) -> Option { - self.service + async fn transaction(&self, id: TxId) -> anyhow::Result> { + Ok(self + .service .find_one(id) - .map(|info| info.tx().clone().deref().into()) + .await + .map_err(|e| anyhow::anyhow!(e))? + .map(|info| info.tx().clone().deref().into())) } - fn submission_time(&self, id: TxId) -> Option { - self.service + async fn submission_time(&self, id: TxId) -> anyhow::Result> { + Ok(self + .service .find_one(id) - .map(|info| Tai64::from_unix(info.submitted_time().as_secs() as i64)) + .await + .map_err(|e| anyhow::anyhow!(e))? + .map(|info| { + Tai64::from_unix( + info.creation_instant() + .duration_since(std::time::UNIX_EPOCH) + .expect("Time can't be lower than 0") + .as_secs() as i64, + ) + })) } - async fn insert( - &self, - txs: Vec>, - ) -> Vec> { + async fn insert(&self, tx: Transaction) -> anyhow::Result<()> { self.service - .insert(txs) + .insert(tx) .await - .into_iter() - .map(|res| res.map_err(|e| anyhow::anyhow!(e))) - .collect() + .map_err(|e| anyhow::anyhow!(e)) } fn tx_update_subscribe( @@ -161,7 +169,7 @@ impl worker::TxPool for TxPoolAdapter { block_height: &BlockHeight, status: TransactionStatus, ) { - self.service.send_complete(id, block_height, status) + self.service.notify_complete_tx(id, block_height, status) } } @@ -215,3 +223,12 @@ impl worker::BlockImporter for GraphQLBlockImporter { self.import_result_provider_adapter.result_at_height(height) } } + +#[async_trait::async_trait] +impl MemoryPool for SharedMemoryPool { + type Memory = MemoryFromPool; + + async fn get_memory(&self) -> Self::Memory { + self.memory_pool.take_raw().await + } +} diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index fdd1d3183bc..d554c7ddc45 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -11,6 +11,7 @@ use crate::{ }, storage::{ contracts::ContractsInfo, + da_compression::DaCompressedBlocks, relayed_transactions::RelayedTransactionStatuses, transactions::OwnedTransactionIndexCursor, }, @@ -22,13 +23,17 @@ use crate::{ }, }; use fuel_core_storage::{ + blueprint::BlueprintInspect, + codec::Encode, iter::{ BoxedIter, IntoBoxedIter, IterDirection, IteratorOverTable, }, + kv_store::KeyValueInspect, not_found, + structured_storage::TableWithBlueprint, transactional::{ IntoTransaction, StorageTransaction, @@ -37,10 +42,6 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsRef, }; -use fuel_core_txpool::types::{ - ContractId, - TxId, -}; use fuel_core_types::{ blockchain::{ block::CompressedBlock, @@ -51,8 +52,10 @@ use fuel_core_types::{ fuel_tx::{ Address, Bytes32, + ContractId, Salt, Transaction, + TxId, TxPointer, UtxoId, }, @@ -69,6 +72,19 @@ impl OffChainDatabase for OffChainIterableKeyValueView { .and_then(|height| height.ok_or(not_found!("BlockHeight"))) } + fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult> { + let column = ::column(); + let encoder = + <::Blueprint as BlueprintInspect< + DaCompressedBlocks, + Self, + >>::KeyCodec::encode(height); + + self.get(encoder.as_ref(), column)? + .ok_or_else(|| not_found!(DaCompressedBlocks)) + .map(|value| value.as_ref().clone()) + } + fn tx_status(&self, tx_id: &TxId) -> StorageResult { self.get_tx_status(tx_id) .transpose() diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index 76532a8d718..b3a6d860e76 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -30,10 +30,6 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsRef, }; -use fuel_core_txpool::types::{ - ContractId, - TxId, -}; use fuel_core_types::{ blockchain::{ block::CompressedBlock, @@ -43,7 +39,9 @@ use fuel_core_types::{ entities::relayer::message::Message, fuel_tx::{ AssetId, + ContractId, Transaction, + TxId, }, fuel_types::{ BlockHeight, diff --git a/crates/fuel-core/src/service/adapters/p2p.rs b/crates/fuel-core/src/service/adapters/p2p.rs index db3545c095b..a01fde29430 100644 --- a/crates/fuel-core/src/service/adapters/p2p.rs +++ b/crates/fuel-core/src/service/adapters/p2p.rs @@ -10,12 +10,12 @@ use fuel_core_p2p::ports::{ }; use fuel_core_services::stream::BoxStream; use fuel_core_storage::Result as StorageResult; -use fuel_core_txpool::types::TxId; use fuel_core_types::{ blockchain::{ consensus::Genesis, SealedBlockHeader, }, + fuel_tx::TxId, fuel_types::BlockHeight, services::p2p::{ NetworkableTransactionPool, @@ -59,19 +59,28 @@ impl BlockHeightImporter for BlockImporterAdapter { } impl TxPool for TxPoolAdapter { - fn get_tx_ids(&self, max_txs: usize) -> Vec { - self.service.get_tx_ids(max_txs) + async fn get_tx_ids(&self, max_txs: usize) -> anyhow::Result> { + self.service + .get_tx_ids(max_txs) + .await + .map_err(|e| anyhow::anyhow!(e)) } - fn get_full_txs(&self, tx_ids: Vec) -> Vec> { - self.service + async fn get_full_txs( + &self, + tx_ids: Vec, + ) -> anyhow::Result>> { + Ok(self + .service .find(tx_ids) + .await + .map_err(|e| anyhow::anyhow!(e))? .into_iter() .map(|tx_info| { tx_info.map(|tx| { NetworkableTransactionPool::PoolTransaction(tx.tx().clone()) }) }) - .collect() + .collect()) } } diff --git a/crates/fuel-core/src/service/adapters/producer.rs b/crates/fuel-core/src/service/adapters/producer.rs index d7ac7590106..7a977f69228 100644 --- a/crates/fuel-core/src/service/adapters/producer.rs +++ b/crates/fuel-core/src/service/adapters/producer.rs @@ -78,12 +78,21 @@ impl BlockProducerAdapter { } } -#[async_trait::async_trait] impl TxPool for TxPoolAdapter { type TxSource = TransactionsSource; - fn get_source(&self, block_height: BlockHeight) -> Self::TxSource { - TransactionsSource::new(self.service.clone(), block_height) + async fn get_source( + &self, + gas_price: u64, + _: BlockHeight, + ) -> anyhow::Result { + let tx_pool = self + .service + .borrow_txpool() + .await + .map_err(|e| anyhow::anyhow!(e))?; + + Ok(TransactionsSource::new(gas_price, tx_pool)) } } diff --git a/crates/fuel-core/src/service/adapters/txpool.rs b/crates/fuel-core/src/service/adapters/txpool.rs index 97ff57fb200..f88d5ca44d4 100644 --- a/crates/fuel-core/src/service/adapters/txpool.rs +++ b/crates/fuel-core/src/service/adapters/txpool.rs @@ -1,14 +1,10 @@ use crate::{ database::OnChainIterableKeyValueView, - service::{ - adapters::{ - BlockImporterAdapter, - ConsensusParametersProvider, - P2PAdapter, - SharedMemoryPool, - StaticGasPrice, - }, - vm_pool::MemoryFromPool, + service::adapters::{ + BlockImporterAdapter, + ConsensusParametersProvider, + P2PAdapter, + StaticGasPrice, }, }; use fuel_core_services::stream::BoxStream; @@ -21,15 +17,10 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsRef, }; -use fuel_core_txpool::{ - ports::{ - BlockImporter, - ConsensusParametersProvider as ConsensusParametersProviderTrait, - GasPriceProvider, - MemoryPool, - }, - types::TxId, - Result as TxPoolResult, +use fuel_core_txpool::ports::{ + BlockImporter, + ConsensusParametersProvider as ConsensusParametersProviderTrait, + GasPriceProvider, }; use fuel_core_types::{ blockchain::header::ConsensusParametersVersion, @@ -41,6 +32,7 @@ use fuel_core_types::{ BlobId, ConsensusParameters, Transaction, + TxId, UtxoId, }, fuel_types::{ @@ -68,9 +60,7 @@ impl BlockImporter for BlockImporterAdapter { #[cfg(feature = "p2p")] #[async_trait::async_trait] -impl fuel_core_txpool::ports::PeerToPeer for P2PAdapter { - type GossipedTransaction = TransactionGossipData; - +impl fuel_core_txpool::ports::NotifyP2P for P2PAdapter { fn broadcast_transaction(&self, transaction: Arc) -> anyhow::Result<()> { if let Some(service) = &self.service { service.broadcast_transaction(transaction) @@ -79,6 +69,23 @@ impl fuel_core_txpool::ports::PeerToPeer for P2PAdapter { } } + fn notify_gossip_transaction_validity( + &self, + message_info: GossipsubMessageInfo, + validity: GossipsubMessageAcceptance, + ) -> anyhow::Result<()> { + if let Some(service) = &self.service { + service.notify_gossip_transaction_validity(message_info, validity) + } else { + Ok(()) + } + } +} + +#[cfg(feature = "p2p")] +impl fuel_core_txpool::ports::P2PSubscriptions for P2PAdapter { + type GossipedTransaction = TransactionGossipData; + fn gossiped_transaction_events(&self) -> BoxStream { use tokio_stream::{ wrappers::BroadcastStream, @@ -108,19 +115,11 @@ impl fuel_core_txpool::ports::PeerToPeer for P2PAdapter { Box::pin(fuel_core_services::stream::pending()) } } +} - fn notify_gossip_transaction_validity( - &self, - message_info: GossipsubMessageInfo, - validity: GossipsubMessageAcceptance, - ) -> anyhow::Result<()> { - if let Some(service) = &self.service { - service.notify_gossip_transaction_validity(message_info, validity) - } else { - Ok(()) - } - } - +#[cfg(feature = "p2p")] +#[async_trait::async_trait] +impl fuel_core_txpool::ports::P2PRequests for P2PAdapter { async fn request_tx_ids(&self, peer_id: PeerId) -> anyhow::Result> { if let Some(service) = &self.service { service.get_all_transactions_ids_from_peer(peer_id).await @@ -145,47 +144,54 @@ impl fuel_core_txpool::ports::PeerToPeer for P2PAdapter { } #[cfg(not(feature = "p2p"))] -#[async_trait::async_trait] -impl fuel_core_txpool::ports::PeerToPeer for P2PAdapter { - type GossipedTransaction = TransactionGossipData; +const _: () = { + #[async_trait::async_trait] + impl fuel_core_txpool::ports::NotifyP2P for P2PAdapter { + fn broadcast_transaction( + &self, + _transaction: Arc, + ) -> anyhow::Result<()> { + Ok(()) + } - fn broadcast_transaction( - &self, - _transaction: Arc, - ) -> anyhow::Result<()> { - Ok(()) + fn notify_gossip_transaction_validity( + &self, + _message_info: GossipsubMessageInfo, + _validity: GossipsubMessageAcceptance, + ) -> anyhow::Result<()> { + Ok(()) + } } - fn gossiped_transaction_events(&self) -> BoxStream { - Box::pin(fuel_core_services::stream::pending()) - } + impl fuel_core_txpool::ports::P2PSubscriptions for P2PAdapter { + type GossipedTransaction = TransactionGossipData; - fn notify_gossip_transaction_validity( - &self, - _message_info: GossipsubMessageInfo, - _validity: GossipsubMessageAcceptance, - ) -> anyhow::Result<()> { - Ok(()) - } + fn gossiped_transaction_events(&self) -> BoxStream { + Box::pin(fuel_core_services::stream::pending()) + } - fn subscribe_new_peers(&self) -> BoxStream { - Box::pin(fuel_core_services::stream::pending()) + fn subscribe_new_peers(&self) -> BoxStream { + Box::pin(fuel_core_services::stream::pending()) + } } - async fn request_tx_ids(&self, _peer_id: PeerId) -> anyhow::Result> { - Ok(vec![]) - } + #[async_trait::async_trait] + impl fuel_core_txpool::ports::P2PRequests for P2PAdapter { + async fn request_tx_ids(&self, _peer_id: PeerId) -> anyhow::Result> { + Ok(vec![]) + } - async fn request_txs( - &self, - _peer_id: PeerId, - _tx_ids: Vec, - ) -> anyhow::Result>> { - Ok(vec![]) + async fn request_txs( + &self, + _peer_id: PeerId, + _tx_ids: Vec, + ) -> anyhow::Result>> { + Ok(vec![]) + } } -} +}; -impl fuel_core_txpool::ports::TxPoolDb for OnChainIterableKeyValueView { +impl fuel_core_txpool::ports::TxPoolPersistentStorage for OnChainIterableKeyValueView { fn utxo(&self, utxo_id: &UtxoId) -> StorageResult> { self.storage::() .get(utxo_id) @@ -209,8 +215,8 @@ impl fuel_core_txpool::ports::TxPoolDb for OnChainIterableKeyValueView { #[async_trait::async_trait] impl GasPriceProvider for StaticGasPrice { - async fn next_gas_price(&self) -> TxPoolResult { - Ok(self.gas_price) + fn next_gas_price(&self) -> u64 { + self.gas_price } } @@ -221,12 +227,3 @@ impl ConsensusParametersProviderTrait for ConsensusParametersProvider { self.shared_state.latest_consensus_parameters_with_version() } } - -#[async_trait::async_trait] -impl MemoryPool for SharedMemoryPool { - type Memory = MemoryFromPool; - - async fn get_memory(&self) -> Self::Memory { - self.memory_pool.take_raw().await - } -} diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 092909929a1..8e17d417e85 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -27,11 +27,15 @@ use fuel_core_p2p::config::{ pub use fuel_core_poa::Trigger; #[cfg(feature = "relayer")] use fuel_core_relayer::Config as RelayerConfig; +use fuel_core_txpool::config::Config as TxPoolConfig; use fuel_core_types::blockchain::header::StateTransitionBytecodeVersion; use crate::{ combined_database::CombinedDatabaseConfig, - graphql_api::ServiceConfig as GraphQLConfig, + graphql_api::{ + worker_service::DaCompressionConfig, + ServiceConfig as GraphQLConfig, + }, }; #[derive(Clone, Debug)] @@ -51,12 +55,13 @@ pub struct Config { pub block_production: Trigger, pub predefined_blocks_path: Option, pub vm: VMConfig, - pub txpool: fuel_core_txpool::Config, + pub txpool: TxPoolConfig, pub block_producer: fuel_core_producer::Config, pub starting_gas_price: u64, pub gas_price_change_percent: u64, pub min_gas_price: u64, pub gas_price_threshold_percent: u64, + pub da_compression: DaCompressionConfig, pub block_importer: fuel_core_importer::Config, #[cfg(feature = "relayer")] pub relayer: Option, @@ -134,8 +139,10 @@ impl Config { 0, ), max_queries_depth: 16, - max_queries_complexity: 20000, + max_queries_complexity: 80000, max_queries_recursive_depth: 16, + max_queries_directives: 10, + max_concurrent_queries: 1024, request_body_bytes_limit: 16 * 1024 * 1024, query_log_threshold_time: Duration::from_secs(2), api_request_timeout: Duration::from_secs(60), @@ -149,14 +156,15 @@ impl Config { block_production: Trigger::Instant, predefined_blocks_path: None, vm: Default::default(), - txpool: fuel_core_txpool::Config { + txpool: TxPoolConfig { utxo_validation, - transaction_ttl: Duration::from_secs(60 * 100000000), - ..fuel_core_txpool::Config::default() + max_txs_ttl: Duration::from_secs(60 * 100000000), + ..Default::default() }, block_producer: fuel_core_producer::Config { ..Default::default() }, + da_compression: DaCompressionConfig::Disabled, starting_gas_price, gas_price_change_percent, min_gas_price, diff --git a/crates/fuel-core/src/service/query.rs b/crates/fuel-core/src/service/query.rs index 9bf0c3823ea..f82f0f9679a 100644 --- a/crates/fuel-core/src/service/query.rs +++ b/crates/fuel-core/src/service/query.rs @@ -1,21 +1,25 @@ //! Queries we can run directly on `FuelService`. -use std::sync::Arc; - +use fuel_core_storage::Result as StorageResult; use fuel_core_types::{ fuel_tx::{ Transaction, UniqueIdentifier, }, fuel_types::Bytes32, - services::txpool::InsertionResult, + services::txpool::TransactionStatus as TxPoolTxStatus, }; use futures::{ Stream, StreamExt, }; +use std::time::SystemTimeError; use crate::{ - query::transaction_status_change, + database::OffChainIterableKeyValueView, + query::{ + transaction_status_change, + TxnStatusChangeState, + }, schema::tx::types::TransactionStatus, }; @@ -23,26 +27,19 @@ use super::*; impl FuelService { /// Submit a transaction to the txpool. - pub async fn submit(&self, tx: Transaction) -> anyhow::Result { - let results: Vec<_> = self - .shared + pub async fn submit(&self, tx: Transaction) -> anyhow::Result<()> { + self.shared .txpool_shared_state - .insert(vec![Arc::new(tx)]) + .insert(tx) .await - .into_iter() - .collect::>() - .map_err(|e| anyhow::anyhow!(e))?; - results - .into_iter() - .next() - .ok_or_else(|| anyhow::anyhow!("Nothing was inserted")) + .map_err(|e| anyhow::anyhow!(e)) } /// Submit a transaction to the txpool and return a stream of status changes. pub async fn submit_and_status_change( &self, tx: Transaction, - ) -> anyhow::Result>> { + ) -> anyhow::Result> + '_> { let id = tx.id(&self .shared .config @@ -50,7 +47,7 @@ impl FuelService { .chain_config() .consensus_parameters .chain_id()); - let stream = self.transaction_status_change(id)?; + let stream = self.transaction_status_change(id).await?; self.submit(tx).await?; Ok(stream) } @@ -67,7 +64,7 @@ impl FuelService { .chain_config() .consensus_parameters .chain_id()); - let stream = self.transaction_status_change(id)?.filter(|status| { + let stream = self.transaction_status_change(id).await?.filter(|status| { futures::future::ready(!matches!(status, Ok(TransactionStatus::Submitted(_)))) }); futures::pin_mut!(stream); @@ -79,20 +76,39 @@ impl FuelService { } /// Return a stream of status changes for a transaction. - pub fn transaction_status_change( + pub async fn transaction_status_change( &self, id: Bytes32, - ) -> anyhow::Result>> { - let txpool = self.shared.txpool_shared_state.clone(); + ) -> anyhow::Result> + '_> { + let txpool = &self.shared.txpool_shared_state; let db = self.shared.database.off_chain().latest_view()?; let rx = txpool.tx_update_subscribe(id)?; - Ok(transaction_status_change( - move |id| match db.get_tx_status(&id)? { - Some(status) => Ok(Some(status)), - None => Ok(txpool.find_one(id).map(Into::into)), - }, - rx, - id, - )) + let state = StatusChangeState { db, txpool }; + Ok(transaction_status_change(state, rx, id).await) + } +} + +struct StatusChangeState<'a> { + db: OffChainIterableKeyValueView, + txpool: &'a TxPoolSharedState, +} + +impl<'a> TxnStatusChangeState for StatusChangeState<'a> { + async fn get_tx_status(&self, id: Bytes32) -> StorageResult> { + match self.db.get_tx_status(&id)? { + Some(status) => Ok(Some(status)), + None => { + let result = self + .txpool + .find_one(id) + .await + .map_err(|e| anyhow::anyhow!(e))?; + let status = result + .map(|status| status.try_into()) + .transpose() + .map_err(|e: SystemTimeError| anyhow::anyhow!(e))?; + Ok(status) + } + } } } diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 93686950f21..13c9f6a9d01 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -65,14 +65,7 @@ pub type PoAService = fuel_core_poa::Service< >; #[cfg(feature = "p2p")] pub type P2PService = fuel_core_p2p::service::Service; -pub type TxPoolSharedState = fuel_core_txpool::service::SharedState< - P2PAdapter, - Database, - ExecutorAdapter, - FuelGasPriceProvider, - ConsensusParametersProvider, - SharedMemoryPool, ->; +pub type TxPoolSharedState = fuel_core_txpool::SharedState; pub type BlockProducerService = fuel_core_producer::block_producer::Producer< Database, TxPoolAdapter, @@ -201,15 +194,15 @@ pub fn init_sub_services( let gas_price_provider = FuelGasPriceProvider::new(gas_price_service_v0.shared.clone()); let txpool = fuel_core_txpool::new_service( + chain_id, config.txpool.clone(), - database.on_chain().clone(), - importer_adapter.clone(), p2p_adapter.clone(), - executor.clone(), + importer_adapter.clone(), + database.on_chain().clone(), + consensus_parameters_provider.clone(), last_height, gas_price_provider.clone(), - consensus_parameters_provider.clone(), - SharedMemoryPool::new(config.memory_pool_size), + executor.clone(), ); let tx_pool_adapter = TxPoolAdapter::new(txpool.shared.clone()); @@ -289,6 +282,7 @@ pub fn init_sub_services( database.on_chain().clone(), database.off_chain().clone(), chain_id, + config.da_compression.clone(), config.continue_on_error, ); @@ -297,8 +291,8 @@ pub fn init_sub_services( utxo_validation: config.utxo_validation, debug: config.debug, vm_backtrace: config.vm.backtrace, - max_tx: config.txpool.max_tx, - max_txpool_depth: config.txpool.max_depth, + max_tx: config.txpool.pool_limits.max_txs, + max_txpool_dependency_chain_length: config.txpool.max_txs_chain_count, chain_name, }; diff --git a/crates/fuel-core/src/state.rs b/crates/fuel-core/src/state.rs index 5f1626e827d..06ee176c652 100644 --- a/crates/fuel-core/src/state.rs +++ b/crates/fuel-core/src/state.rs @@ -8,18 +8,10 @@ use crate::{ }; use fuel_core_storage::{ iter::{ - BoxedIter, - IntoBoxedIter, IterDirection, IterableStore, }, - kv_store::{ - KVItem, - KeyValueInspect, - StorageColumn, - Value, - WriteOperation, - }, + kv_store::StorageColumn, transactional::Changes, Result as StorageResult, }; @@ -99,88 +91,3 @@ where unimplemented!() } } - -/// A type that allows to iterate over the `Changes`. -pub struct ChangesIterator<'a, Description> { - changes: &'a Changes, - _marker: core::marker::PhantomData, -} - -impl<'a, Description> ChangesIterator<'a, Description> { - /// Creates a new instance of the `ChangesIterator`. - pub fn new(changes: &'a Changes) -> Self { - Self { - changes, - _marker: Default::default(), - } - } -} - -impl<'a, Description> KeyValueInspect for ChangesIterator<'a, Description> -where - Description: DatabaseDescription, -{ - type Column = Description::Column; - - fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { - Ok(self - .changes - .get(&column.id()) - .and_then(|tree| tree.get(key)) - .and_then(|operation| match operation { - WriteOperation::Insert(value) => Some(value.clone()), - WriteOperation::Remove => None, - })) - } -} - -impl<'a, Description> IterableStore for ChangesIterator<'a, Description> -where - Description: DatabaseDescription, -{ - fn iter_store( - &self, - column: Self::Column, - prefix: Option<&[u8]>, - start: Option<&[u8]>, - direction: IterDirection, - ) -> BoxedIter { - if let Some(tree) = self.changes.get(&column.id()) { - fuel_core_storage::iter::iterator(tree, prefix, start, direction) - .filter_map(|(key, value)| match value { - WriteOperation::Insert(value) => { - Some((key.clone().into(), value.clone())) - } - WriteOperation::Remove => None, - }) - .map(Ok) - .into_boxed() - } else { - core::iter::empty().into_boxed() - } - } - - fn iter_store_keys( - &self, - column: Self::Column, - prefix: Option<&[u8]>, - start: Option<&[u8]>, - direction: IterDirection, - ) -> BoxedIter { - // We cannot define iter_store_keys appropriately for the `ChangesIterator`, - // because we have to filter out the keys that were removed, which are - // marked as `WriteOperation::Remove` in the value - // copied as-is from the above function, but only to return keys - if let Some(tree) = self.changes.get(&column.id()) { - fuel_core_storage::iter::iterator(tree, prefix, start, direction) - .filter_map(|(key, value)| match value { - WriteOperation::Insert(_) => Some(key.clone().into()), - WriteOperation::Remove => None, - }) - .map(Ok) - .into_boxed() - } else { - core::iter::empty().into_boxed() - } - } -} diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index bbf42e40038..8306df1b1fb 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -10,17 +10,22 @@ use fuel_core_storage::{ Value, }, structured_storage::StructuredStorage, + tables::BlobData, Error as StorageError, Mappable, MerkleRoot, MerkleRootStorage, + PredicateStorageRequirements, Result as StorageResult, StorageAsRef, StorageInspect, StorageRead, StorageSize, }; -use std::borrow::Cow; +use std::{ + borrow::Cow, + fmt::Debug, +}; #[derive(Debug, Clone)] pub struct GenericDatabase { @@ -176,3 +181,13 @@ impl core::ops::DerefMut for GenericDatabase { self.as_mut() } } + +impl PredicateStorageRequirements for GenericDatabase +where + Self: StorageRead, + Error: Debug, +{ + fn storage_error_to_string(error: Error) -> String { + format!("{:?}", error) + } +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs index 32194a716a2..6b4c82f09ff 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs @@ -307,10 +307,10 @@ pub fn draw_profit( .x_label_area_size(40) .y_label_area_size(100) .right_y_label_area_size(100) - .build_cartesian_2d(0..actual_profit_gwei.len(), min..max) + .build_cartesian_2d(0..projected_profit.len(), min..max) .unwrap() .set_secondary_coord( - 0..actual_profit_gwei.len(), + 0..projected_profit.len(), 0..*pessimistic_block_costs_gwei.iter().max().unwrap(), ); diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 6ad906cc41f..94d7c06faa3 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -1,10 +1,10 @@ -use fuel_gas_price_algorithm::v1::{ - AlgorithmUpdaterV1, - RecordedBlock, -}; -use std::num::NonZeroU64; - use super::*; +use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; +use std::{ + collections::BTreeMap, + num::NonZeroU64, + ops::Range, +}; pub mod da_cost_per_byte; @@ -24,6 +24,13 @@ pub struct Simulator { da_cost_per_byte: Vec, } +// (usize, ((u64, u64), &'a Option<(Range, u128) +struct BlockData { + fullness: u64, + bytes: u64, + maybe_da_block: Option<(Range, u128)>, +} + impl Simulator { pub fn new(da_cost_per_byte: Vec) -> Self { Simulator { da_cost_per_byte } @@ -49,7 +56,14 @@ impl Simulator { &fullness_and_bytes, ); - let blocks = l2_blocks.zip(da_blocks.iter()).enumerate(); + let blocks = l2_blocks + .zip(da_blocks.iter()) + .map(|((fullness, bytes), maybe_da_block)| BlockData { + fullness, + bytes, + maybe_da_block: maybe_da_block.clone(), + }) + .enumerate(); let updater = self.build_updater(da_p_component, da_d_component); @@ -90,7 +104,7 @@ impl Simulator { latest_da_cost_per_byte: 0, projected_total_da_cost: 0, latest_known_total_da_cost_excess: 0, - unrecorded_blocks: vec![], + unrecorded_blocks: BTreeMap::new(), da_p_component, da_d_component, last_profit: 0, @@ -104,8 +118,7 @@ impl Simulator { capacity: u64, max_block_bytes: u64, fullness_and_bytes: Vec<(u64, u64)>, - // blocks: Enumerate, Iter>>>>, - blocks: impl Iterator>))>, + blocks: impl Iterator, mut updater: AlgorithmUpdaterV1, ) -> SimulationResults { let mut gas_prices = vec![]; @@ -115,13 +128,18 @@ impl Simulator { let mut projected_cost_totals = vec![]; let mut actual_costs = vec![]; let mut pessimistic_costs = vec![]; - for (index, ((fullness, bytes), da_block)) in blocks { + for (index, block_data) in blocks { + let BlockData { + fullness, + bytes, + maybe_da_block: da_block, + } = block_data; let height = index as u32 + 1; exec_gas_prices.push(updater.new_scaled_exec_price); da_gas_prices.push(updater.new_scaled_da_gas_price); let gas_price = updater.algorithm().calculate(); gas_prices.push(gas_price); - let total_fee = gas_price * fullness; + let total_fee = gas_price as u128 * fullness as u128; updater .update_l2_block_data( height, @@ -137,13 +155,13 @@ impl Simulator { projected_cost_totals.push(updater.projected_total_da_cost); // Update DA blocks on the occasion there is one - if let Some(da_blocks) = &da_block { - let mut total_cost = updater.latest_known_total_da_cost_excess; - for block in da_blocks { - total_cost += block.block_cost as u128; - actual_costs.push(total_cost); + if let Some((range, cost)) = da_block { + for height in range.to_owned() { + updater + .update_da_record_data(height..(height + 1), cost) + .unwrap(); + actual_costs.push(updater.latest_known_total_da_cost_excess) } - updater.update_da_record_data(&da_blocks).unwrap(); } } let (fullness_without_capacity, bytes): (Vec<_>, Vec<_>) = @@ -155,7 +173,7 @@ impl Simulator { let bytes_and_costs: Vec<_> = bytes .iter() .zip(self.da_cost_per_byte.iter()) - .map(|(bytes, cost_per_byte)| (*bytes, (*bytes * cost_per_byte) as u64)) + .map(|(bytes, da_cost_per_byte)| (*bytes, (*bytes * da_cost_per_byte) as u64)) .collect(); let actual_profit: Vec = actual_costs @@ -187,7 +205,7 @@ impl Simulator { da_recording_rate: usize, da_finalization_rate: usize, fullness_and_bytes: &Vec<(u64, u64)>, - ) -> Vec>> { + ) -> Vec, u128)>> { let l2_blocks_with_no_da_blocks = std::iter::repeat(None).take(da_finalization_rate); let (_, da_blocks) = fullness_and_bytes @@ -199,12 +217,9 @@ impl Simulator { |(mut delayed, mut recorded), (index, ((_fullness, bytes), cost_per_byte))| { let total_cost = *bytes * cost_per_byte; + let height = index as u32 + 1; - let converted = RecordedBlock { - height, - block_bytes: *bytes, - block_cost: total_cost as u64, - }; + let converted = (height, bytes, total_cost); delayed.push(converted); if delayed.len() == da_recording_rate { recorded.push(Some(delayed)); @@ -215,7 +230,16 @@ impl Simulator { } }, ); - l2_blocks_with_no_da_blocks.chain(da_blocks).collect() + let da_block_ranges = da_blocks.into_iter().map(|maybe_recorded_blocks| { + maybe_recorded_blocks.map(|list| { + let heights_iter = list.iter().map(|(height, _, _)| *height); + let min = heights_iter.clone().min().unwrap(); + let max = heights_iter.max().unwrap(); + let cost: u128 = list.iter().map(|(_, _, cost)| *cost as u128).sum(); + (min..(max + 1), cost) + }) + }); + l2_blocks_with_no_da_blocks.chain(da_block_ranges).collect() } } diff --git a/crates/fuel-gas-price-algorithm/src/v0.rs b/crates/fuel-gas-price-algorithm/src/v0.rs index 494c9045a88..418e6619c7e 100644 --- a/crates/fuel-gas-price-algorithm/src/v0.rs +++ b/crates/fuel-gas-price-algorithm/src/v0.rs @@ -66,12 +66,6 @@ pub struct AlgorithmUpdaterV0 { pub l2_block_fullness_threshold_percent: u64, } -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] -pub struct BlockBytes { - pub height: u32, - pub block_bytes: u64, -} - impl AlgorithmUpdaterV0 { pub fn new( new_exec_price: u64, diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index dad6d64beea..a1a5f7dd536 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -1,11 +1,14 @@ +use crate::utils::cumulative_percentage_change; use std::{ cmp::max, + collections::BTreeMap, num::NonZeroU64, - ops::Div, + ops::{ + Div, + Range, + }, }; -use crate::utils::cumulative_percentage_change; - #[cfg(test)] mod tests; @@ -16,9 +19,11 @@ pub enum Error { #[error("Skipped DA block update: expected {expected:?}, got {got:?}")] SkippedDABlock { expected: u32, got: u32 }, #[error("Could not calculate cost per byte: {bytes:?} bytes, {cost:?} cost")] - CouldNotCalculateCostPerByte { bytes: u64, cost: u64 }, + CouldNotCalculateCostPerByte { bytes: u128, cost: u128 }, #[error("Failed to include L2 block data: {0}")] FailedTooIncludeL2BlockData(String), + #[error("L2 block expected but not found in unrecorded blocks: {0}")] + L2BlockExpectedNotFound(u32), } #[derive(Debug, Clone, PartialEq)] @@ -94,6 +99,8 @@ impl AlgorithmV1 { /// The DA portion also uses a moving average of the profits over the last `avg_window` blocks /// instead of the actual profit. Setting the `avg_window` to 1 will effectively disable the /// moving average. +type Height = u32; +type Bytes = u64; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] pub struct AlgorithmUpdaterV1 { // Execution @@ -144,8 +151,9 @@ pub struct AlgorithmUpdaterV1 { pub second_to_last_profit: i128, /// The latest known cost per byte for recording blocks on the DA chain pub latest_da_cost_per_byte: u128, + /// The unrecorded blocks that are used to calculate the projected cost of recording blocks - pub unrecorded_blocks: Vec, + pub unrecorded_blocks: BTreeMap, } /// A value that represents a value between 0 and 100. Higher values are clamped to 100 @@ -176,29 +184,16 @@ impl core::ops::Deref for ClampedPercentage { } } -#[derive(Debug, Clone)] -pub struct RecordedBlock { - pub height: u32, - pub block_bytes: u64, - pub block_cost: u64, -} - -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] -pub struct BlockBytes { - pub height: u32, - pub block_bytes: u64, -} - impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, - blocks: &[RecordedBlock], + height_range: Range, + range_cost: u128, ) -> Result<(), Error> { - for block in blocks { - self.da_block_update(block.height, block.block_bytes, block.block_cost)?; + if !height_range.is_empty() { + self.da_block_update(height_range, range_cost)?; + self.recalculate_projected_cost(); } - self.recalculate_projected_cost(); - self.normalize_rewards_and_costs(); Ok(()) } @@ -208,7 +203,7 @@ impl AlgorithmUpdaterV1 { used: u64, capacity: NonZeroU64, block_bytes: u64, - fee_wei: u64, + fee_wei: u128, ) -> Result<(), Error> { let expected = self.l2_block_height.saturating_add(1); if height != expected { @@ -220,12 +215,14 @@ impl AlgorithmUpdaterV1 { self.l2_block_height = height; // rewards - self.update_rewards(fee_wei); + self.update_da_rewards(fee_wei); let rewards = self.clamped_rewards_as_i128(); // costs - self.update_projected_cost(block_bytes); + self.update_projected_da_cost(block_bytes); let projected_total_da_cost = self.clamped_projected_cost_as_i128(); + + // profit let last_profit = rewards.saturating_sub(projected_total_da_cost); self.update_last_profit(last_profit); @@ -234,22 +231,18 @@ impl AlgorithmUpdaterV1 { self.update_da_gas_price(); // metadata - self.unrecorded_blocks.push(BlockBytes { - height, - block_bytes, - }); + self.unrecorded_blocks.insert(height, block_bytes); Ok(()) } } - fn update_rewards(&mut self, fee_wei: u64) { + fn update_da_rewards(&mut self, fee_wei: u128) { let block_da_reward = self.da_portion_of_fee(fee_wei); - self.total_da_rewards_excess = self - .total_da_rewards_excess - .saturating_add(block_da_reward.into()); + self.total_da_rewards_excess = + self.total_da_rewards_excess.saturating_add(block_da_reward); } - fn update_projected_cost(&mut self, block_bytes: u64) { + fn update_projected_da_cost(&mut self, block_bytes: u64) { let block_projected_da_cost = (block_bytes as u128).saturating_mul(self.latest_da_cost_per_byte); self.projected_total_da_cost = self @@ -258,12 +251,11 @@ impl AlgorithmUpdaterV1 { } // Take the `fee_wei` and return the portion of the fee that should be used for paying DA costs - fn da_portion_of_fee(&self, fee_wei: u64) -> u64 { + fn da_portion_of_fee(&self, fee_wei: u128) -> u128 { // fee_wei * (da_price / (exec_price + da_price)) - let numerator = fee_wei.saturating_mul(self.descaled_da_price()); - let denominator = self - .descaled_exec_price() - .saturating_add(self.descaled_da_price()); + let numerator = fee_wei.saturating_mul(self.descaled_da_price() as u128); + let denominator = (self.descaled_exec_price() as u128) + .saturating_add(self.descaled_da_price() as u128); if denominator == 0 { 0 } else { @@ -374,45 +366,61 @@ impl AlgorithmUpdaterV1 { fn da_block_update( &mut self, - height: u32, - block_bytes: u64, - block_cost: u64, + height_range: Range, + range_cost: u128, ) -> Result<(), Error> { let expected = self.da_recorded_block_height.saturating_add(1); - if height != expected { + let first = height_range.start; + if first != expected { Err(Error::SkippedDABlock { - expected: self.da_recorded_block_height.saturating_add(1), - got: height, + expected, + got: first, }) } else { - let new_cost_per_byte: u128 = (block_cost as u128) - .checked_div(block_bytes as u128) - .ok_or(Error::CouldNotCalculateCostPerByte { - bytes: block_bytes, - cost: block_cost, - })?; - self.da_recorded_block_height = height; - let new_block_cost = self + let last = height_range.end.saturating_sub(1); + let range_bytes = self.drain_l2_block_bytes_for_range(height_range)?; + let new_cost_per_byte: u128 = range_cost.checked_div(range_bytes).ok_or( + Error::CouldNotCalculateCostPerByte { + bytes: range_bytes, + cost: range_cost, + }, + )?; + self.da_recorded_block_height = last; + let new_da_block_cost = self .latest_known_total_da_cost_excess - .saturating_add(block_cost as u128); - self.latest_known_total_da_cost_excess = new_block_cost; + .saturating_add(range_cost); + self.latest_known_total_da_cost_excess = new_da_block_cost; self.latest_da_cost_per_byte = new_cost_per_byte; Ok(()) } } + fn drain_l2_block_bytes_for_range( + &mut self, + height_range: Range, + ) -> Result { + let mut total: u128 = 0; + for expected_height in height_range { + let (actual_height, bytes) = self + .unrecorded_blocks + .pop_first() + .ok_or(Error::L2BlockExpectedNotFound(expected_height))?; + if actual_height != expected_height { + return Err(Error::L2BlockExpectedNotFound(expected_height)); + } + total = total.saturating_add(bytes as u128); + } + Ok(total) + } + fn recalculate_projected_cost(&mut self) { - // remove all blocks that have been recorded - self.unrecorded_blocks - .retain(|block| block.height > self.da_recorded_block_height); // add the cost of the remaining blocks let projection_portion: u128 = self .unrecorded_blocks .iter() - .map(|block| { - (block.block_bytes as u128).saturating_mul(self.latest_da_cost_per_byte) - }) - .sum(); + .map(|(_, &bytes)| (bytes as u128)) + .fold(0_u128, |acc, n| acc.saturating_add(n)) + .saturating_mul(self.latest_da_cost_per_byte); self.projected_total_da_cost = self .latest_known_total_da_cost_excess .saturating_add(projection_portion); @@ -435,34 +443,4 @@ impl AlgorithmUpdaterV1 { for_height: self.l2_block_height, } } - - // We only need to track the difference between the rewards and costs after we have true DA data - // Normalize, or zero out the lower value and subtract it from the higher value - fn normalize_rewards_and_costs(&mut self) { - let (excess, projected_cost_excess) = - if self.total_da_rewards_excess > self.latest_known_total_da_cost_excess { - ( - self.total_da_rewards_excess - .saturating_sub(self.latest_known_total_da_cost_excess), - self.projected_total_da_cost - .saturating_sub(self.latest_known_total_da_cost_excess), - ) - } else { - ( - self.latest_known_total_da_cost_excess - .saturating_sub(self.total_da_rewards_excess), - self.projected_total_da_cost - .saturating_sub(self.total_da_rewards_excess), - ) - }; - - self.projected_total_da_cost = projected_cost_excess; - if self.total_da_rewards_excess > self.latest_known_total_da_cost_excess { - self.total_da_rewards_excess = excess; - self.latest_known_total_da_cost_excess = 0; - } else { - self.total_da_rewards_excess = 0; - self.latest_known_total_da_cost_excess = excess; - } - } } diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index 707173baca4..0aca738a3f0 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -2,10 +2,7 @@ #![allow(clippy::arithmetic_side_effects)] #![allow(clippy::cast_possible_truncation)] -use crate::v1::{ - AlgorithmUpdaterV1, - BlockBytes, -}; +use crate::v1::AlgorithmUpdaterV1; #[cfg(test)] mod algorithm_v1_tests; @@ -14,6 +11,12 @@ mod update_da_record_data_tests; #[cfg(test)] mod update_l2_block_data_tests; +#[derive(Debug, Clone)] +pub struct BlockBytes { + pub height: u32, + pub block_bytes: u64, +} + pub struct UpdaterBuilder { min_exec_gas_price: u64, min_da_gas_price: u64, @@ -175,7 +178,11 @@ impl UpdaterBuilder { latest_da_cost_per_byte: self.da_cost_per_byte, projected_total_da_cost: self.project_total_cost, latest_known_total_da_cost_excess: self.latest_known_total_cost, - unrecorded_blocks: self.unrecorded_blocks, + unrecorded_blocks: self + .unrecorded_blocks + .iter() + .map(|b| (b.height, b.block_bytes)) + .collect(), last_profit: self.last_profit, second_to_last_profit: self.second_to_last_profit, min_da_gas_price: self.min_da_gas_price, diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index e9d64a63bcc..123520b4e99 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -1,74 +1,67 @@ use crate::v1::{ - tests::UpdaterBuilder, - BlockBytes, + tests::{ + BlockBytes, + UpdaterBuilder, + }, Error, - RecordedBlock, }; -use proptest::{ - prelude::Rng, - prop_compose, - proptest, -}; -use rand::SeedableRng; #[test] fn update_da_record_data__increases_block() { // given let da_recorded_block_height = 0; - let mut updater = UpdaterBuilder::new() - .with_da_recorded_block_height(da_recorded_block_height) - .build(); - - let blocks = vec![ - RecordedBlock { + let recorded_range = 1u32..3; + let recorded_cost = 200; + let unrecorded_blocks = vec![ + BlockBytes { height: 1, block_bytes: 1000, - block_cost: 100, }, - RecordedBlock { + BlockBytes { height: 2, - block_bytes: 1000, - block_cost: 100, + block_bytes: 2000, }, ]; + let mut updater = UpdaterBuilder::new() + .with_da_recorded_block_height(da_recorded_block_height) + .with_unrecorded_blocks(unrecorded_blocks) + .build(); // when - updater.update_da_record_data(&blocks).unwrap(); + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); // then let expected = 2; let actual = updater.da_recorded_block_height; - assert_eq!(actual, expected); + assert_eq!(expected, actual); } #[test] fn update_da_record_data__throws_error_if_out_of_order() { // given let da_recorded_block_height = 0; + let bad_recorded_range = 2u32..4; + let recorded_cost = 200; + let unrecorded_blocks = vec![BlockBytes { + height: 1, + block_bytes: 1000, + }]; let mut updater = UpdaterBuilder::new() .with_da_recorded_block_height(da_recorded_block_height) + .with_unrecorded_blocks(unrecorded_blocks) .build(); - let blocks = vec![ - RecordedBlock { - height: 1, - block_bytes: 1000, - block_cost: 100, - }, - RecordedBlock { - height: 3, - block_bytes: 1000, - block_cost: 100, - }, - ]; - // when - let actual_error = updater.update_da_record_data(&blocks).unwrap_err(); + let actual_error = updater + .update_da_record_data(bad_recorded_range, recorded_cost) + .unwrap_err(); // then let expected_error = Error::SkippedDABlock { - expected: 2, - got: 3, + expected: 1, + got: 2, }; assert_eq!(actual_error, expected_error); } @@ -77,20 +70,23 @@ fn update_da_record_data__throws_error_if_out_of_order() { fn update_da_record_data__updates_cost_per_byte() { // given let da_cost_per_byte = 20; + let block_bytes = 1000; + let unrecorded_blocks = vec![BlockBytes { + height: 1, + block_bytes, + }]; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) + .with_unrecorded_blocks(unrecorded_blocks) .build(); - let block_bytes = 1000; let new_cost_per_byte = 100; - let block_cost = block_bytes * new_cost_per_byte; - let blocks = vec![RecordedBlock { - height: 1, - block_bytes, - block_cost, - }]; + let recorded_cost = (block_bytes * new_cost_per_byte) as u128; + let recorded_range = 1u32..2; // when - updater.update_da_record_data(&blocks).unwrap(); + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); // then let expected = new_cost_per_byte as u128; @@ -106,39 +102,39 @@ fn update_da_record_data__updates_known_total_cost() { let l2_block_height = 15; let projected_total_cost = 2000; let known_total_cost = 1500; + let unrecorded_blocks = vec![ + BlockBytes { + height: 11, + block_bytes: 1000, + }, + BlockBytes { + height: 12, + block_bytes: 2000, + }, + BlockBytes { + height: 13, + block_bytes: 1500, + }, + ]; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost) .with_known_total_cost(known_total_cost) + .with_unrecorded_blocks(unrecorded_blocks) .build(); - let block_bytes = 1000; - let block_cost = 100; - let blocks = vec![ - RecordedBlock { - height: 11, - block_bytes, - block_cost, - }, - RecordedBlock { - height: 12, - block_bytes, - block_cost, - }, - RecordedBlock { - height: 13, - block_bytes, - block_cost, - }, - ]; + let recorded_range = 11u32..14; + let recorded_cost = 300; // when - updater.update_da_record_data(&blocks).unwrap(); + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); // then let actual = updater.latest_known_total_da_cost_excess; - let expected = known_total_cost + (3 * block_cost as u128); + let expected = known_total_cost + recorded_cost; assert_eq!(actual, expected); } @@ -181,25 +177,13 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let block_bytes = 1000; let new_cost_per_byte = 100; let block_cost = block_bytes * new_cost_per_byte; - let blocks = vec![ - RecordedBlock { - height: 11, - block_bytes, - block_cost, - }, - RecordedBlock { - height: 12, - block_bytes, - block_cost, - }, - RecordedBlock { - height: 13, - block_bytes, - block_cost, - }, - ]; + + let recorded_range = 11u32..14; + let recorded_cost = block_cost * 3; // when - updater.update_da_record_data(&blocks).unwrap(); + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); // then assert_eq!(updater.l2_block_height, updater.da_recorded_block_height); @@ -216,7 +200,8 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let da_cost_per_byte = 20; let da_recorded_block_height = 10; let l2_block_height = 15; - let known_total_cost = 1500; + let original_known_total_cost = 1500; + let block_bytes = 1000; let mut unrecorded_blocks = vec![ BlockBytes { height: 11, @@ -235,54 +220,49 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let remaining = vec![ BlockBytes { height: 14, - block_bytes: 1200, + block_bytes, }, BlockBytes { height: 15, - block_bytes: 3000, + block_bytes, }, ]; + let to_be_removed = unrecorded_blocks.clone(); unrecorded_blocks.extend(remaining.clone()); let guessed_cost: u64 = unrecorded_blocks .iter() .map(|block| block.block_bytes * da_cost_per_byte) .sum(); - let projected_total_cost = known_total_cost + guessed_cost; + let projected_total_cost = original_known_total_cost + guessed_cost; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte as u128) .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) - .with_known_total_cost(known_total_cost as u128) + .with_known_total_cost(original_known_total_cost as u128) .with_unrecorded_blocks(unrecorded_blocks) .build(); - let block_bytes = 1000; let new_cost_per_byte = 100; - let block_cost = block_bytes * new_cost_per_byte; - let blocks = vec![ - RecordedBlock { - height: 11, - block_bytes, - block_cost, - }, - RecordedBlock { - height: 12, - block_bytes, - block_cost, - }, - RecordedBlock { - height: 13, - block_bytes, - block_cost, - }, - ]; + let (recorded_heights, recorded_cost) = + to_be_removed + .iter() + .fold((vec![], 0), |(mut range, cost), block| { + range.push(block.height); + (range, cost + block.block_bytes * new_cost_per_byte) + }); + let min = recorded_heights.iter().min().unwrap(); + let max = recorded_heights.iter().max().unwrap(); + let recorded_range = *min..(max + 1); + // when - updater.update_da_record_data(&blocks).unwrap(); + updater + .update_da_record_data(recorded_range, recorded_cost as u128) + .unwrap(); // then let actual = updater.projected_total_da_cost; - let new_known_total_cost = known_total_cost + 3 * block_cost; + let new_known_total_cost = original_known_total_cost + recorded_cost; let guessed_part: u64 = remaining .iter() .map(|block| block.block_bytes * new_cost_per_byte) @@ -290,127 +270,3 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let expected = new_known_total_cost + guessed_part; assert_eq!(actual, expected as u128); } - -prop_compose! { - fn arb_vec_of_da_blocks()(last_da_block: u32, count in 1..123usize, rng_seed: u64) -> Vec { - let rng = &mut rand::rngs::StdRng::seed_from_u64(rng_seed); - let mut blocks = Vec::with_capacity(count); - for i in 0..count { - let block_bytes = rng.gen_range(100..131_072); - let cost_per_byte = rng.gen_range(1..1000000); - let block_cost = block_bytes * cost_per_byte; - blocks.push(RecordedBlock { - height: last_da_block + 1 + i as u32, - block_bytes, - block_cost, - }); - } - blocks - } -} - -prop_compose! { - fn reward_greater_than_cost_with_da_blocks()(cost: u64, extra: u64, blocks in arb_vec_of_da_blocks()) -> (u128, u128, Vec) { - let cost_from_blocks = blocks.iter().map(|block| block.block_cost as u128).sum::(); - let reward = cost as u128 + cost_from_blocks + extra as u128; - (cost as u128, reward, blocks) - } -} - -proptest! { - #[test] - fn update_da_record_data__when_reward_is_greater_than_cost_will_zero_cost_and_subtract_from_reward( - (cost, reward, blocks) in reward_greater_than_cost_with_da_blocks() - ) { - _update_da_record_data__when_reward_is_greater_than_cost_will_zero_cost_and_subtract_from_reward( - cost, - reward, - blocks - ) - } -} - -fn _update_da_record_data__when_reward_is_greater_than_cost_will_zero_cost_and_subtract_from_reward( - known_total_cost: u128, - total_rewards: u128, - blocks: Vec, -) { - // given - let da_cost_per_byte = 20; - let da_recorded_block_height = blocks.first().unwrap().height - 1; - let l2_block_height = 15; - let mut updater = UpdaterBuilder::new() - .with_da_cost_per_byte(da_cost_per_byte) - .with_da_recorded_block_height(da_recorded_block_height) - .with_l2_block_height(l2_block_height) - .with_known_total_cost(known_total_cost) - .with_total_rewards(total_rewards) - .build(); - - let new_costs = blocks.iter().map(|block| block.block_cost).sum::(); - - // when - updater.update_da_record_data(&blocks).unwrap(); - - // then - let expected = total_rewards - new_costs as u128 - known_total_cost; - let actual = updater.total_da_rewards_excess; - assert_eq!(actual, expected); - - let expected = 0; - let actual = updater.latest_known_total_da_cost_excess; - assert_eq!(actual, expected); -} - -prop_compose! { - fn cost_greater_than_reward_with_da_blocks()(reward: u64, extra: u64, blocks in arb_vec_of_da_blocks()) -> (u128, u128, Vec) { - let cost_from_blocks = blocks.iter().map(|block| block.block_cost as u128).sum::(); - let cost = reward as u128 + cost_from_blocks + extra as u128; - (cost, reward as u128, blocks) - } -} - -proptest! { - #[test] - fn update_da_record_data__when_cost_is_greater_than_reward_will_zero_reward_and_subtract_from_cost( - (cost, reward, blocks) in cost_greater_than_reward_with_da_blocks() - ) { - _update_da_record_data__when_cost_is_greater_than_reward_will_zero_reward_and_subtract_from_cost( - cost, - reward, - blocks - ) - } -} - -fn _update_da_record_data__when_cost_is_greater_than_reward_will_zero_reward_and_subtract_from_cost( - known_total_cost: u128, - total_rewards: u128, - blocks: Vec, -) { - // given - let da_cost_per_byte = 20; - let da_recorded_block_height = blocks.first().unwrap().height - 1; - let l2_block_height = 15; - let mut updater = UpdaterBuilder::new() - .with_da_cost_per_byte(da_cost_per_byte) - .with_da_recorded_block_height(da_recorded_block_height) - .with_l2_block_height(l2_block_height) - .with_known_total_cost(known_total_cost) - .with_total_rewards(total_rewards) - .build(); - - let new_costs = blocks.iter().map(|block| block.block_cost).sum::(); - - // when - updater.update_da_record_data(&blocks).unwrap(); - - // then - let expected = 0; - let actual = updater.total_da_rewards_excess; - assert_eq!(actual, expected); - - let expected = known_total_cost + new_costs as u128 - total_rewards; - let actual = updater.latest_known_total_da_cost_excess; - assert_eq!(actual, expected); -} diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index 1ad9a1aacad..6d5e4cf2ce7 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -1,6 +1,8 @@ use crate::v1::{ - tests::UpdaterBuilder, - BlockBytes, + tests::{ + BlockBytes, + UpdaterBuilder, + }, Error, }; @@ -104,10 +106,10 @@ fn update_l2_block_data__updates_the_total_reward_value() { .unwrap(); // then - let expected = (fee * starting_da_gas_price) - .div_ceil(starting_da_gas_price + starting_exec_gas_price); + let expected = (fee * starting_da_gas_price as u128) + .div_ceil(starting_da_gas_price as u128 + starting_exec_gas_price as u128); let actual = updater.total_da_rewards_excess; - assert_eq!(actual, expected as u128); + assert_eq!(actual, expected); } #[test] @@ -480,7 +482,7 @@ fn update_l2_block_data__even_profit_maintains_price() { 50, 100.try_into().unwrap(), block_bytes, - total_fee, + total_fee.into(), ) .unwrap(); let algo = updater.algorithm(); @@ -563,8 +565,9 @@ fn update_l2_block_data__adds_l2_block_to_unrecorded_blocks() { height, block_bytes, }; - let contains_block_bytes = updater.unrecorded_blocks.contains(&block_bytes); - assert!(contains_block_bytes); + let expected = block_bytes.block_bytes; + let actual = updater.unrecorded_blocks.get(&block_bytes.height).unwrap(); + assert_eq!(expected, *actual); } #[test] @@ -598,10 +601,12 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded height, block_bytes, }; - let contains_block_bytes = updater.unrecorded_blocks.contains(&block_bytes); + let contains_block_bytes = + updater.unrecorded_blocks.contains_key(&block_bytes.height); assert!(contains_block_bytes); - let contains_preexisting_block_bytes = - updater.unrecorded_blocks.contains(&preexisting_block); + let contains_preexisting_block_bytes = updater + .unrecorded_blocks + .contains_key(&preexisting_block.height); assert!(contains_preexisting_block_bytes); } diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 77140da452d..345bcd64287 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -15,11 +15,15 @@ async-trait = { workspace = true } fuel-core-metrics = { workspace = true } futures = { workspace = true } parking_lot = { workspace = true } +rayon = { workspace = true, optional = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } [dev-dependencies] +fuel-core-services = { path = ".", features = ["sync-processor"] } +futures = { workspace = true } mockall = { workspace = true } [features] test-helpers = [] +sync-processor = ["dep:rayon"] diff --git a/crates/services/consensus_module/poa/Cargo.toml b/crates/services/consensus_module/poa/Cargo.toml index eb97756e4eb..5d71379853d 100644 --- a/crates/services/consensus_module/poa/Cargo.toml +++ b/crates/services/consensus_module/poa/Cargo.toml @@ -29,6 +29,7 @@ aws-config = { version = "1.1.7", features = ["behavior-version-latest"] } fuel-core-poa = { path = ".", features = ["test-helpers"] } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { path = "./../../../storage", features = ["test-helpers"] } +fuel-core-trace = { path = "./../../../trace" } fuel-core-types = { path = "./../../../types", features = ["test-helpers"] } mockall = { workspace = true } rand = { workspace = true } diff --git a/crates/services/consensus_module/poa/src/lib.rs b/crates/services/consensus_module/poa/src/lib.rs index 272c569ed61..19dfeb826cf 100644 --- a/crates/services/consensus_module/poa/src/lib.rs +++ b/crates/services/consensus_module/poa/src/lib.rs @@ -24,3 +24,6 @@ pub use service::{ new_service, Service, }; + +#[cfg(test)] +fuel_core_trace::enable_tracing!(); diff --git a/crates/services/consensus_module/poa/src/ports.rs b/crates/services/consensus_module/poa/src/ports.rs index 2148bcfb44b..af3e889eff9 100644 --- a/crates/services/consensus_module/poa/src/ports.rs +++ b/crates/services/consensus_module/poa/src/ports.rs @@ -10,10 +10,7 @@ use fuel_core_types::{ header::BlockHeader, primitives::DaBlockHeight, }, - fuel_tx::{ - Transaction, - TxId, - }, + fuel_tx::Transaction, fuel_types::{ BlockHeight, Bytes32, @@ -23,11 +20,7 @@ use fuel_core_types::{ BlockImportInfo, UncommittedResult as UncommittedImportResult, }, - executor::{ - Error as ExecutorError, - UncommittedResult as UncommittedExecutionResult, - }, - txpool::ArcPoolTx, + executor::UncommittedResult as UncommittedExecutionResult, }, tai64::Tai64, }; @@ -35,14 +28,9 @@ use std::collections::HashMap; #[cfg_attr(test, mockall::automock)] pub trait TransactionPool: Send + Sync { - /// Returns the number of pending transactions in the `TxPool`. - fn pending_number(&self) -> usize; - - fn total_consumable_gas(&self) -> u64; - - fn remove_txs(&self, tx_ids: Vec<(TxId, ExecutorError)>) -> Vec; + fn new_txs_watcher(&self) -> tokio::sync::watch::Receiver<()>; - fn transaction_status_events(&self) -> BoxStream; + fn notify_skipped_txs(&self, tx_ids_and_reasons: Vec<(Bytes32, String)>); } /// The source of transactions for the block. diff --git a/crates/services/consensus_module/poa/src/service.rs b/crates/services/consensus_module/poa/src/service.rs index 2d2f07a21d7..04255ac108a 100644 --- a/crates/services/consensus_module/poa/src/service.rs +++ b/crates/services/consensus_module/poa/src/service.rs @@ -16,7 +16,6 @@ use tokio::{ Instant, }, }; -use tokio_stream::StreamExt; use crate::{ ports::{ @@ -37,10 +36,7 @@ use crate::{ Trigger, }; use fuel_core_services::{ - stream::{ - BoxFuture, - BoxStream, - }, + stream::BoxFuture, RunnableService, RunnableTask, Service as OtherService, @@ -132,7 +128,7 @@ pub struct MainTask { block_producer: B, block_importer: I, txpool: T, - tx_status_update_stream: BoxStream, + new_txs_watcher: tokio::sync::watch::Receiver<()>, request_receiver: mpsc::Receiver, shared_state: SharedState, last_height: BlockHeight, @@ -164,7 +160,7 @@ where predefined_blocks: PB, clock: C, ) -> Self { - let tx_status_update_stream = txpool.transaction_status_events(); + let new_txs_watcher = txpool.new_txs_watcher(); let (request_sender, request_receiver) = mpsc::channel(1024); let (last_height, last_timestamp, last_block_created) = Self::extract_block_info(clock.now(), last_block); @@ -194,7 +190,7 @@ where txpool, block_producer, block_importer, - tx_status_update_stream, + new_txs_watcher, request_receiver, shared_state: SharedState { request_sender }, last_height, @@ -343,16 +339,18 @@ where .await? .into(); - let mut tx_ids_to_remove = Vec::with_capacity(skipped_transactions.len()); - for (tx_id, err) in skipped_transactions { - tracing::error!( + if !skipped_transactions.is_empty() { + let mut tx_ids_to_remove = Vec::with_capacity(skipped_transactions.len()); + for (tx_id, err) in skipped_transactions { + tracing::error!( "During block production got invalid transaction {:?} with error {:?}", tx_id, err ); - tx_ids_to_remove.push((tx_id, err)); + tx_ids_to_remove.push((tx_id, err.to_string())); + } + self.txpool.notify_skipped_txs(tx_ids_to_remove); } - self.txpool.remove_txs(tx_ids_to_remove); // Sign the block and seal it let seal = self.signer.seal_block(&block).await?; @@ -438,16 +436,9 @@ where Ok(()) } - pub(crate) async fn on_txpool_event(&mut self) -> anyhow::Result<()> { + async fn on_txpool_event(&mut self) -> anyhow::Result<()> { match self.trigger { - Trigger::Instant => { - let pending_number = self.txpool.pending_number(); - // skip production if there are no pending transactions - if pending_number > 0 { - self.produce_next_block().await?; - } - Ok(()) - } + Trigger::Instant => self.produce_next_block().await, Trigger::Never | Trigger::Interval { .. } => Ok(()), } } @@ -531,19 +522,14 @@ where let should_continue; let mut sync_state = self.sync_task_handle.shared.clone(); // make sure we're synced first - while *sync_state.borrow_and_update() == SyncState::NotSynced { + if *sync_state.borrow_and_update() == SyncState::NotSynced { tokio::select! { biased; result = watcher.while_started() => { should_continue = result?.started(); return Ok(should_continue); } - _ = sync_state.changed() => { - break; - } - _ = self.tx_status_update_stream.next() => { - // ignore txpool events while syncing - } + _ = sync_state.changed() => {} } } @@ -587,19 +573,9 @@ where should_continue = false; } } - // TODO: This should likely be refactored to use something like tokio::sync::Notify. - // Otherwise, if a bunch of txs are submitted at once and all the txs are included - // into the first block production trigger, we'll still call the event handler - // for each tx after they've already been included into a block. - // The poa service also doesn't care about events unrelated to new tx submissions, - // and shouldn't be awoken when txs are completed or squeezed out of the pool. - txpool_event = self.tx_status_update_stream.next() => { - if txpool_event.is_some() { - self.on_txpool_event().await.context("While processing txpool event")?; - should_continue = true; - } else { - should_continue = false; - } + _ = self.new_txs_watcher.changed() => { + self.on_txpool_event().await.context("While processing txpool event")?; + should_continue = true; } _ = next_block_production => { match self.on_timer().await.context("While processing timer event") { diff --git a/crates/services/consensus_module/poa/src/service_test.rs b/crates/services/consensus_module/poa/src/service_test.rs index 6b04e339a2b..5230e03f42b 100644 --- a/crates/services/consensus_module/poa/src/service_test.rs +++ b/crates/services/consensus_module/poa/src/service_test.rs @@ -23,7 +23,6 @@ use crate::{ use async_trait::async_trait; use fuel_core_chain_config::default_consensus_dev_key; use fuel_core_services::{ - stream::pending, Service as StorageTrait, ServiceRunner, State, @@ -41,10 +40,7 @@ use fuel_core_types::{ SealedBlock, }, fuel_crypto::SecretKey, - fuel_tx::{ - field::ScriptGasLimit, - *, - }, + fuel_tx::*, fuel_types::{ BlockHeight, ChainId, @@ -140,7 +136,7 @@ impl TestContextBuilder { self } - fn build(self) -> TestContext { + async fn build(self) -> TestContext { let config = self.config.unwrap_or_default(); let producer = self.producer.unwrap_or_else(|| { let mut producer = MockBlockProducer::default(); @@ -192,7 +188,7 @@ impl TestContextBuilder { predefined_blocks, watch, ); - service.start().unwrap(); + service.start_and_await().await.unwrap(); TestContext { service, time } } } @@ -239,74 +235,35 @@ impl TestContext { pub struct TxPoolContext { pub txpool: MockTransactionPool, pub txs: Arc>>, - pub status_sender: Arc>>, + pub new_txs_notifier: watch::Sender<()>, } impl MockTransactionPool { fn no_tx_updates() -> Self { let mut txpool = MockTransactionPool::default(); - txpool - .expect_transaction_status_events() - .returning(|| Box::pin(pending())); + txpool.expect_new_txs_watcher().returning({ + let (sender, _) = watch::channel(()); + move || sender.subscribe() + }); + txpool.expect_notify_skipped_txs().returning(|_| {}); txpool } pub fn new_with_txs(txs: Vec