feat(ethexe): impl basis for runtime related RPC calls; impl `calcula…

…te_reply_for_handle` v1 (#4238)

Cargo.toml

@@ -143,6 +143,7 @@ hashbrown = "0.14.5"
 hex = { version = "0.4.3", default-features = false }
 hex-literal = "0.4.1"
 impl-trait-for-tuples = "0.2.2"
+impl-serde = "0.4.0"
 jsonrpsee = { version = "^0.16" }
 libc = { version = "0.2", default-features = false }
 log = { version = "0.4.22", default-features = false }

core/Cargo.toml

@@ -38,6 +38,7 @@ primitive-types = { workspace = true, features = ["scale-info"] }
 
 # Optional dependencies
 serde = { workspace = true, features = ["derive"], optional = true }
+impl-serde = { workspace = true, optional = true }
 
 [dev-dependencies]
 wabt.workspace = true
@@ -49,4 +50,4 @@ numerated = { workspace = true, features = ["mock"] }
 [features]
 default = []
 strict = []
-std = ["serde/std", "wasmparser/std", "gear-core-errors/serde"]
+std = ["serde/std", "dep:impl-serde", "wasmparser/std", "gear-core-errors/serde"]

core/src/message/mod.rs

@@ -239,6 +239,7 @@ pub trait Packet {
 #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))]
 pub struct ReplyInfo {
     /// Payload of the reply.
+    #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))]
     pub payload: Vec<u8>,
     /// Value sent with reply.
     pub value: u128,

ethexe/cli/src/service.rs

@@ -322,7 +322,7 @@ impl Service {
         db.set_block_end_state_is_valid(block_hash, true);
 
         let header = db.block_header(block_hash).expect("must be set; qed");
-        db.set_latest_valid_block_height(header.height);
+        db.set_latest_valid_block(block_hash, header);
 
         Ok(transition_outcomes)
     }

ethexe/common/src/db.rs

@@ -71,8 +71,8 @@ pub trait BlockMetaStorage: Send + Sync {
     fn block_outcome(&self, block_hash: H256) -> Option<Vec<StateTransition>>;
     fn set_block_outcome(&self, block_hash: H256, outcome: Vec<StateTransition>);
 
-    fn latest_valid_block_height(&self) -> Option<u32>;
-    fn set_latest_valid_block_height(&self, block_height: u32);
+    fn latest_valid_block(&self) -> Option<(H256, BlockHeader)>;
+    fn set_latest_valid_block(&self, block_hash: H256, header: BlockHeader);
 }
 
 pub trait CodesStorage: Send + Sync {

ethexe/db/src/database.rs

@@ -20,7 +20,10 @@
 
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 
-use crate::{CASDatabase, KVDatabase};
+use crate::{
+    overlay::{CASOverlay, KVOverlay},
+    CASDatabase, KVDatabase,
+};
 use ethexe_common::{
     db::{BlockHeader, BlockMetaStorage, CodesStorage},
     router::StateTransition,
@@ -253,18 +256,19 @@ impl BlockMetaStorage for Database {
         );
     }
 
-    fn latest_valid_block_height(&self) -> Option<u32> {
+    fn latest_valid_block(&self) -> Option<(H256, BlockHeader)> {
         self.kv
             .get(&KeyPrefix::LatestValidBlock.one(self.router_address))
-            .map(|block_height| {
-                u32::from_le_bytes(block_height.try_into().expect("must be correct; qed"))
+            .map(|data| {
+                <(H256, BlockHeader)>::decode(&mut data.as_slice())
+                    .expect("Failed to decode data into `(H256, BlockHeader)`")
             })
     }
 
-    fn set_latest_valid_block_height(&self, block_height: u32) {
+    fn set_latest_valid_block(&self, block_hash: H256, header: BlockHeader) {
         self.kv.put(
             &KeyPrefix::LatestValidBlock.one(self.router_address),
-            block_height.to_le_bytes().to_vec(),
+            (block_hash, header).encode(),
         );
     }
 }
@@ -383,6 +387,16 @@ impl Database {
         }
     }
 
+    /// # Safety
+    /// Not ready for using in prod. Intended to be for rpc calls only.
+    pub unsafe fn overlaid(self) -> Self {
+        Self {
+            cas: Box::new(CASOverlay::new(self.cas)),
+            kv: Box::new(KVOverlay::new(self.kv)),
+            router_address: self.router_address,
+        }
+    }
+
     // TODO: temporary solution for MVP runtime-interfaces db access.
     pub fn read_by_hash(&self, hash: H256) -> Option<Vec<u8>> {
         self.cas.read(&hash)

ethexe/db/src/lib.rs

@@ -23,6 +23,7 @@ use gprimitives::H256;
 
 mod database;
 mod mem;
+mod overlay;
 mod rocks;
 
 pub use database::Database;

ethexe/db/src/overlay.rs

@@ -0,0 +1,105 @@
+// This file is part of Gear.
+//
+// Copyright (C) 2024 Gear Technologies Inc.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{CASDatabase, KVDatabase, MemDb};
+use gear_core::ids::hash;
+use gprimitives::H256;
+use std::collections::HashSet;
+
+pub struct CASOverlay {
+    db: Box<dyn CASDatabase>,
+    mem: MemDb,
+}
+
+impl CASOverlay {
+    pub fn new(db: Box<dyn CASDatabase>) -> Self {
+        Self {
+            db,
+            mem: MemDb::default(),
+        }
+    }
+}
+
+impl CASDatabase for CASOverlay {
+    fn clone_boxed(&self) -> Box<dyn CASDatabase> {
+        Box::new(Self {
+            db: self.db.clone_boxed(),
+            mem: self.mem.clone(),
+        })
+    }
+
+    fn read(&self, hash: &H256) -> Option<Vec<u8>> {
+        self.mem.read(hash).or_else(|| self.db.read(hash))
+    }
+
+    fn write_by_hash(&self, hash: &H256, data: &[u8]) {
+        self.mem.write_by_hash(hash, data)
+    }
+}
+
+pub struct KVOverlay {
+    db: Box<dyn KVDatabase>,
+    mem: MemDb,
+}
+
+impl KVOverlay {
+    pub fn new(db: Box<dyn KVDatabase>) -> Self {
+        Self {
+            db,
+            mem: MemDb::default(),
+        }
+    }
+}
+
+impl KVDatabase for KVOverlay {
+    fn clone_boxed_kv(&self) -> Box<dyn KVDatabase> {
+        Box::new(Self {
+            db: self.db.clone_boxed_kv(),
+            mem: self.mem.clone(),
+        })
+    }
+
+    fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
+        self.mem.get(key).or_else(|| self.db.get(key))
+    }
+
+    fn take(&self, _key: &[u8]) -> Option<Vec<u8>> {
+        unimplemented!()
+    }
+
+    fn put(&self, key: &[u8], value: Vec<u8>) {
+        self.mem.put(key, value)
+    }
+
+    fn iter_prefix<'a>(
+        &'a self,
+        prefix: &'a [u8],
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + '_> {
+        let mem_iter = self.mem.iter_prefix(prefix);
+        let db_iter = self.db.iter_prefix(prefix);
+
+        let full_iter = mem_iter.chain(db_iter);
+
+        let mut known_keys = HashSet::new();
+
+        let filtered_iter =
+            full_iter.filter_map(move |(k, v)| known_keys.insert(hash(&k)).then_some((k, v)));
+
+        Box::new(filtered_iter)
+    }
+}

ethexe/observer/src/query.rs

@@ -72,10 +72,9 @@ impl Query {
             .set_block_end_program_states(hash, Default::default());
 
         // set latest valid if empty.
-        if self.database.latest_valid_block_height().is_none() {
+        if self.database.latest_valid_block().is_none() {
             let genesis_header = self.get_block_header_meta(hash).await?;
-            self.database
-                .set_latest_valid_block_height(genesis_header.height);
+            self.database.set_latest_valid_block(hash, genesis_header);
         }
 
         Ok(())
@@ -160,7 +159,8 @@ impl Query {
         let current_block = self.get_block_header_meta(block_hash).await?;
         let latest_valid_block_height = self
             .database
-            .latest_valid_block_height()
+            .latest_valid_block()
+            .map(|(_, header)| header.height)
             .expect("genesis by default; qed");
 
         if current_block.height >= latest_valid_block_height
@@ -196,15 +196,21 @@ impl Query {
 
         // Continue loading chain by parent hashes from the current block to the latest valid block.
         let mut hash = block_hash;
-        let mut height = current_block.height;
+
         while hash != self.genesis_block_hash {
             // If the block's end state is valid, set it as the latest valid block
             if self
                 .database
                 .block_end_state_is_valid(hash)
                 .unwrap_or(false)
             {
-                self.database.set_latest_valid_block_height(height);
+                let header = match headers_map.get(&hash) {
+                    Some(header) => header.clone(),
+                    None => self.get_block_header_meta(hash).await?,
+                };
+
+                self.database.set_latest_valid_block(hash, header);
+
                 log::trace!("Nearest valid in db block found: {hash}");
                 break;
             }
@@ -218,7 +224,6 @@ impl Query {
                 Some(header) => header.parent_hash,
                 None => self.get_block_parent_hash(hash).await?,
             };
-            height -= 1;
         }
 
         let mut actual_commitment_queue: VecDeque<H256> = self

ethexe/processor/src/host/mod.rs

@@ -43,7 +43,6 @@ pub type Store = wasmtime::Store<StoreData>;
 
 #[derive(Clone)]
 pub(crate) struct InstanceCreator {
-    db: Database,
     engine: wasmtime::Engine,
     instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
 
@@ -54,7 +53,7 @@ pub(crate) struct InstanceCreator {
 }
 
 impl InstanceCreator {
-    pub fn new(db: Database, runtime: Vec<u8>) -> Result<Self> {
+    pub fn new(runtime: Vec<u8>) -> Result<Self> {
         gear_runtime_interface::sandbox_init();
 
         let engine = wasmtime::Engine::default();
@@ -72,7 +71,6 @@ impl InstanceCreator {
         let instance_pre = Arc::new(instance_pre);
 
         Ok(Self {
-            db,
             engine,
             instance_pre,
             chain_head: None,
@@ -87,7 +85,6 @@ impl InstanceCreator {
         let mut instance_wrapper = InstanceWrapper {
             instance,
             store,
-            db: self.db().clone(),
             chain_head: self.chain_head,
         };
 
@@ -100,10 +97,6 @@ impl InstanceCreator {
         Ok(instance_wrapper)
     }
 
-    pub fn db(&self) -> &Database {
-        &self.db
-    }
-
     pub fn set_chain_head(&mut self, chain_head: H256) {
         self.chain_head = Some(chain_head);
     }
@@ -112,15 +105,10 @@ impl InstanceCreator {
 pub(crate) struct InstanceWrapper {
     instance: wasmtime::Instance,
     store: Store,
-    db: Database,
     chain_head: Option<H256>,
 }
 
 impl InstanceWrapper {
-    pub fn db(&self) -> &Database {
-        &self.db
-    }
-
     #[allow(unused)]
     pub fn data(&self) -> &StoreData {
         self.store.data()
@@ -139,13 +127,14 @@ impl InstanceWrapper {
 
     pub fn run(
         &mut self,
+        db: Database,
         program_id: ProgramId,
         original_code_id: CodeId,
         state_hash: H256,
         maybe_instrumented_code: Option<InstrumentedCode>,
     ) -> Result<Vec<JournalNote>> {
         let chain_head = self.chain_head.expect("chain head must be set before run");
-        threads::set(self.db.clone(), chain_head, state_hash);
+        threads::set(db, chain_head, state_hash);
 
         let arg = (
             program_id,
@@ -157,9 +146,14 @@ impl InstanceWrapper {
         self.call("run", arg.encode())
     }
 
-    pub fn wake_messages(&mut self, program_id: ProgramId, state_hash: H256) -> Result<H256> {
+    pub fn wake_messages(
+        &mut self,
+        db: Database,
+        program_id: ProgramId,
+        state_hash: H256,
+    ) -> Result<H256> {
         let chain_head = self.chain_head.expect("chain head must be set before wake");
-        threads::set(self.db.clone(), chain_head, state_hash);
+        threads::set(db, chain_head, state_hash);
 
         self.call("wake_messages", (program_id, state_hash).encode())
     }