Merge branch 'master' into mc/chore/p2p-upgrades

.cargo/audit.toml

@@ -1,4 +1,2 @@
 [advisories]
-ignore = [
-    "RUSTSEC-2024-0336" # https://github.com/FuelLabs/fuel-core/issues/1843
-]
+ignore = []

CHANGELOG.md

@@ -8,11 +8,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 - [2366](https://github.com/FuelLabs/fuel-core/pull/2366): The `importer_gas_price_for_block` metric is properly collected.
+- [2369](https://github.com/FuelLabs/fuel-core/pull/2369): The `transaction_insertion_time_in_thread_pool_milliseconds` metric is properly collected.
+
+### Changed
+
+- [2378](https://github.com/FuelLabs/fuel-core/pull/2378): Use cached hash of the topic instead of calculating it on each publishing gossip message.
 
 ### Added
 - [2321](https://github.com/FuelLabs/fuel-core/pull/2321): New metrics for the txpool: "The size of transactions in the txpool" (`txpool_tx_size`), "The time spent by a transaction in the txpool in seconds" (`txpool_tx_time_in_txpool_seconds`), The number of transactions in the txpool (`txpool_number_of_transactions`), "The number of transactions pending verification before entering the txpool" (`txpool_number_of_transactions_pending_verification`), "The number of executable transactions in the txpool" (`txpool_number_of_executable_transactions`), "The time it took to select transactions for inclusion in a block in nanoseconds" (`txpool_select_transaction_time_nanoseconds`), The time it took to insert a transaction in the txpool in milliseconds (`txpool_insert_transaction_time_milliseconds`).
 - [2150](https://github.com/FuelLabs/fuel-core/pull/2150): Upgraded `libp2p` to `0.54.1` and introduced `max_pending_incoming_connections` , `max_established_connections` and `max_pending_outgoing_connections` to limit the number of pending connections.
 
+- [2347](https://github.com/FuelLabs/fuel-core/pull/2364): Add activity concept in order to protect against infinitely increasing DA gas price scenarios
+- [2362](https://github.com/FuelLabs/fuel-core/pull/2362): Added a new request_response protocol version `/fuel/req_res/0.0.2`. In comparison with `/fuel/req/0.0.1`, which returns an empty response when a request cannot be fulfilled, this version returns more meaningful error codes. Nodes still support the version `0.0.1` of the protocol to guarantee backward compatibility with fuel-core nodes. Empty responses received from nodes using the old protocol `/fuel/req/0.0.1` are automatically converted into an error `ProtocolV1EmptyResponse` with error code 0, which is also the only error code implemented. More specific error codes will be added in the future.
+- [2386](https://github.com/FuelLabs/fuel-core/pull/2386): Add a flag to define the maximum number of file descriptors that RocksDB can use. By default it's half of the OS limit.
 
 ## [Version 0.40.0]
 

benches/benches/vm_set/blockchain.rs

@@ -75,6 +75,7 @@ impl BenchDb {
             tmp_dir.path(),
             None,
             Default::default(),
+            -1,
         )
         .unwrap();
         let db = Arc::new(db);

benches/src/db_lookup_times_utils/mod.rs

@@ -27,7 +27,7 @@ mod tests {
 
     fn setup_test_db() -> RocksDb<BenchDatabase> {
         let temp_dir = ShallowTempDir::new();
-        RocksDb::default_open(temp_dir.path(), None).unwrap()
+        RocksDb::default_open(temp_dir.path(), None, -1).unwrap()
     }
 
     #[test]

benches/src/db_lookup_times_utils/utils.rs

@@ -39,7 +39,7 @@ pub fn get_random_block_height(
 pub fn open_rocks_db<Description: DatabaseDescription>(
     path: &Path,
 ) -> Result<RocksDb<Description>> {
-    let db = RocksDb::default_open(path, None)?;
+    let db = RocksDb::default_open(path, None, -1)?;
     Ok(db)
 }
 

bin/fuel-core/Cargo.toml

@@ -35,6 +35,7 @@ hex = { workspace = true }
 humantime = "2.1"
 pyroscope = "0.5"
 pyroscope_pprofrs = "0.2"
+rlimit = "0.10.2"
 serde_json = { workspace = true }
 tikv-jemallocator = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

bin/fuel-core/src/cli/rollback.rs

@@ -5,6 +5,10 @@ use fuel_core::{
     combined_database::CombinedDatabase,
     state::historical_rocksdb::StateRewindPolicy,
 };
+use rlimit::{
+    getrlimit,
+    Resource,
+};
 use std::path::PathBuf;
 
 /// Rollbacks the state of the blockchain to a specific block height.
@@ -19,11 +23,28 @@ pub struct Command {
     )]
     pub database_path: PathBuf,
 
+    /// Defines a specific number of file descriptors that RocksDB can use.
+    ///
+    /// If defined as -1 no limit will be applied and will use the OS limits.
+    /// If not defined the system default divided by two is used.
+    #[clap(
+        long = "rocksdb-max-fds",
+        env,
+        default_value = get_default_max_fds().to_string()
+    )]
+    pub rocksdb_max_fds: i32,
+
     /// The path to the database.
     #[clap(long = "target-block-height")]
     pub target_block_height: u32,
 }
 
+fn get_default_max_fds() -> i32 {
+    getrlimit(Resource::NOFILE)
+        .map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX))
+        .expect("Our supported platforms should return max FD.")
+}
+
 pub async fn exec(command: Command) -> anyhow::Result<()> {
     use crate::cli::ShutdownListener;
 
@@ -32,6 +53,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> {
         path,
         64 * 1024 * 1024,
         StateRewindPolicy::RewindFullRange,
+        command.rocksdb_max_fds,
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}"))?;

bin/fuel-core/src/cli/run.rs

@@ -65,6 +65,10 @@ use pyroscope_pprofrs::{
     pprof_backend,
     PprofConfig,
 };
+use rlimit::{
+    getrlimit,
+    Resource,
+};
 use std::{
     env,
     net,
@@ -126,6 +130,18 @@ pub struct Command {
     )]
     pub database_type: DbType,
 
+    #[cfg(feature = "rocksdb")]
+    /// Defines a specific number of file descriptors that RocksDB can use.
+    ///
+    /// If defined as -1 no limit will be applied and will use the OS limits.
+    /// If not defined the system default divided by two is used.
+    #[clap(
+        long = "rocksdb-max-fds",
+        env,
+        default_value = get_default_max_fds().to_string()
+    )]
+    pub rocksdb_max_fds: i32,
+
     #[cfg(feature = "rocksdb")]
     /// Defines the state rewind policy for the database when RocksDB is enabled.
     ///
@@ -273,6 +289,8 @@ impl Command {
             database_path,
             database_type,
             #[cfg(feature = "rocksdb")]
+            rocksdb_max_fds,
+            #[cfg(feature = "rocksdb")]
             state_rewind_duration,
             db_prune,
             snapshot,
@@ -441,6 +459,8 @@ impl Command {
             max_database_cache_size,
             #[cfg(feature = "rocksdb")]
             state_rewind_policy,
+            #[cfg(feature = "rocksdb")]
+            max_fds: rocksdb_max_fds,
         };
 
         let block_importer = fuel_core::service::config::fuel_core_importer::Config::new(
@@ -591,6 +611,13 @@ impl Command {
     }
 }
 
+#[cfg(feature = "rocksdb")]
+fn get_default_max_fds() -> i32 {
+    getrlimit(Resource::NOFILE)
+        .map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX))
+        .expect("Our supported platforms should return max FD.")
+}
+
 pub async fn get_service_with_shutdown_listeners(
     command: Command,
 ) -> anyhow::Result<(FuelService, ShutdownListener)> {

bin/fuel-core/src/cli/snapshot.rs

@@ -10,6 +10,10 @@ use fuel_core::{
     types::fuel_types::ContractId,
 };
 use fuel_core_chain_config::ChainConfig;
+use rlimit::{
+    getrlimit,
+    Resource,
+};
 use std::path::{
     Path,
     PathBuf,
@@ -29,6 +33,17 @@ pub struct Command {
     )]
     pub database_path: PathBuf,
 
+    /// Defines a specific number of file descriptors that RocksDB can use.
+    ///
+    /// If defined as -1 no limit will be applied and will use the OS limits.
+    /// If not defined the system default divided by two is used.
+    #[clap(
+        long = "rocksdb-max-fds",
+        env,
+        default_value = get_default_max_fds().to_string()
+    )]
+    pub rocksdb_max_fds: i32,
+
     /// Where to save the snapshot
     #[arg(name = "OUTPUT_DIR", long = "output-directory")]
     pub output_dir: PathBuf,
@@ -112,6 +127,12 @@ pub enum SubCommands {
     },
 }
 
+fn get_default_max_fds() -> i32 {
+    getrlimit(Resource::NOFILE)
+        .map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX))
+        .expect("Our supported platforms should return max FD.")
+}
+
 #[cfg(feature = "rocksdb")]
 pub async fn exec(command: Command) -> anyhow::Result<()> {
     use fuel_core::service::genesis::Exporter;
@@ -125,6 +146,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> {
     let db = open_db(
         &command.database_path,
         Some(command.max_database_cache_size),
+        command.rocksdb_max_fds,
     )?;
     let output_dir = command.output_dir;
     let shutdown_listener = ShutdownListener::spawn();
@@ -180,11 +202,16 @@ fn load_chain_config_or_use_testnet(path: Option<&Path>) -> anyhow::Result<Chain
     }
 }
 
-fn open_db(path: &Path, capacity: Option<usize>) -> anyhow::Result<CombinedDatabase> {
+fn open_db(
+    path: &Path,
+    capacity: Option<usize>,
+    max_fds: i32,
+) -> anyhow::Result<CombinedDatabase> {
     CombinedDatabase::open(
         path,
         capacity.unwrap_or(1024 * 1024 * 1024),
         StateRewindPolicy::NoRewind,
+        max_fds,
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}",))
@@ -668,7 +695,8 @@ mod tests {
         let db_path = temp_dir.path().join("db");
         std::fs::create_dir(&db_path)?;
 
-        let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2));
+        let mut db =
+            DbPopulator::new(open_db(&db_path, None, 512)?, StdRng::seed_from_u64(2));
         let state = db.given_persisted_data();
         db.flush();
 
@@ -681,6 +709,7 @@ mod tests {
                 chain_config: None,
                 encoding_command: Some(EncodingCommand::Encoding { encoding }),
             },
+            rocksdb_max_fds: 512,
         });
 
         // Because the test_case macro doesn't work with async tests
@@ -720,7 +749,8 @@ mod tests {
 
         let snapshot_dir = temp_dir.path().join("snapshot");
         let db_path = temp_dir.path().join("db");
-        let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2));
+        let mut db =
+            DbPopulator::new(open_db(&db_path, None, 512)?, StdRng::seed_from_u64(2));
 
         let state = db.given_persisted_data();
         db.flush();
@@ -739,6 +769,7 @@ mod tests {
                     },
                 }),
             },
+            rocksdb_max_fds: 512,
         });
 
         tokio::runtime::Runtime::new()
@@ -763,7 +794,8 @@ mod tests {
         let snapshot_dir = temp_dir.path().join("snapshot");
 
         let db_path = temp_dir.path().join("db");
-        let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2));
+        let mut db =
+            DbPopulator::new(open_db(&db_path, None, 512)?, StdRng::seed_from_u64(2));
 
         let original_state = db.given_persisted_data().sorted().into_state_config();
 
@@ -789,6 +821,7 @@ mod tests {
             output_dir: snapshot_dir.clone(),
             max_database_cache_size: DEFAULT_DATABASE_CACHE_SIZE,
             subcommand: SubCommands::Contract { contract_id },
+            rocksdb_max_fds: 512,
         })
         .await?;
 

crates/fuel-core/src/combined_database.rs

@@ -39,6 +39,8 @@ pub struct CombinedDatabaseConfig {
     pub max_database_cache_size: usize,
     #[cfg(feature = "rocksdb")]
     pub state_rewind_policy: StateRewindPolicy,
+    #[cfg(feature = "rocksdb")]
+    pub max_fds: i32,
 }
 
 /// A database that combines the on-chain, off-chain and relayer databases into one entity.
@@ -79,13 +81,22 @@ impl CombinedDatabase {
         path: &std::path::Path,
         capacity: usize,
         state_rewind_policy: StateRewindPolicy,
+        max_fds: i32,
     ) -> crate::database::Result<Self> {
+        // Split the fds in equitable manner between the databases
+        let max_fds = match max_fds {
+            -1 => -1,
+            _ => max_fds.saturating_div(4),
+        };
         // TODO: Use different cache sizes for different databases
-        let on_chain = Database::open_rocksdb(path, capacity, state_rewind_policy)?;
-        let off_chain = Database::open_rocksdb(path, capacity, state_rewind_policy)?;
+        let on_chain =
+            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
+        let off_chain =
+            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
         let relayer =
-            Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind)?;
-        let gas_price = Database::open_rocksdb(path, capacity, state_rewind_policy)?;
+            Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind, max_fds)?;
+        let gas_price =
+            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
         Ok(Self {
             on_chain,
             off_chain,
@@ -115,6 +126,7 @@ impl CombinedDatabase {
                         &config.database_path,
                         config.max_database_cache_size,
                         config.state_rewind_policy,
+                        config.max_fds,
                     )?
                 }
             }

crates/fuel-core/src/database.rs

@@ -199,12 +199,14 @@ where
         path: &Path,
         capacity: impl Into<Option<usize>>,
         state_rewind_policy: StateRewindPolicy,
+        max_fds: i32,
     ) -> Result<Self> {
         use anyhow::Context;
         let db = HistoricalRocksDB::<Description>::default_open(
             path,
             capacity.into(),
             state_rewind_policy,
+            max_fds,
         )
         .map_err(Into::<anyhow::Error>::into)
         .with_context(|| {
@@ -1076,6 +1078,7 @@ mod tests {
             temp_dir.path(),
             1024 * 1024 * 1024,
             Default::default(),
+            512,
         )
         .unwrap();
         // rocks db fails