From c007e51623a687aa47c1a42e53268adf7783af38 Mon Sep 17 00:00:00 2001 From: Santiago Carmuega Date: Tue, 13 Feb 2024 16:57:41 -0300 Subject: [PATCH] simplify --- Cargo.lock | 121 ++++++++++++++++++++++------------------- Cargo.toml | 4 +- src/serve/grpc/sync.rs | 45 +++------------ src/sync/roll.rs | 28 +++++----- 4 files changed, 88 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0485a7e4..1fd4d4e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1405,6 +1405,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -1912,16 +1921,15 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "pallas" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "pallas-addresses", "pallas-applying", - "pallas-codec 0.22.0", + "pallas-codec 0.23.0", "pallas-configs", - "pallas-crypto 0.22.0", + "pallas-crypto 0.23.0", "pallas-hardano", - "pallas-network 0.22.0", + "pallas-network 0.23.0", "pallas-primitives", "pallas-rolldb", "pallas-traverse", @@ -1932,28 +1940,26 @@ dependencies = [ [[package]] name = "pallas-addresses" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "base58", "bech32 0.9.1", "crc", "hex", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "sha3", "thiserror", ] [[package]] name = "pallas-applying" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "hex", "pallas-addresses", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "pallas-primitives", "pallas-traverse", "rand", @@ -1973,8 +1979,7 @@ dependencies = [ [[package]] name = "pallas-codec" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "hex", "minicbor 0.20.0", @@ -1984,14 +1989,13 @@ dependencies = [ [[package]] name = "pallas-configs" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "base64 0.21.5", "hex", "pallas-addresses", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "serde", "serde_json", ] @@ -2012,12 +2016,11 @@ dependencies = [ [[package]] name = "pallas-crypto" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "cryptoxide", "hex", - "pallas-codec 0.22.0", + "pallas-codec 0.23.0", "rand_core 0.6.4", "serde", "thiserror", @@ -2025,11 +2028,10 @@ dependencies = [ [[package]] name = "pallas-hardano" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "binary-layout", - "pallas-network 0.22.0", + "pallas-network 0.23.0", "pallas-traverse", "tap", "thiserror", @@ -2044,7 +2046,7 @@ checksum = "69d5fb0899c37f74b2b3366d44eef85d44aea2b1cd8e9548263977baa252d9f4" dependencies = [ "byteorder", "hex", - "itertools", + "itertools 0.10.5", "pallas-codec 0.20.0", "pallas-crypto 0.20.0", "thiserror", @@ -2054,14 +2056,13 @@ dependencies = [ [[package]] name = "pallas-network" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "byteorder", "hex", - "itertools", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "itertools 0.12.1", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "rand", "socket2 0.5.5", "thiserror", @@ -2071,29 +2072,27 @@ dependencies = [ [[package]] name = "pallas-primitives" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "base58", "bech32 0.9.1", "hex", "log", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "serde", "serde_json", ] [[package]] name = "pallas-rolldb" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "async-stream", "bincode", "futures-core", "futures-util", - "pallas-crypto 0.22.0", + "pallas-crypto 0.23.0", "rocksdb", "serde", "thiserror", @@ -2103,13 +2102,12 @@ dependencies = [ [[package]] name = "pallas-traverse" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "hex", "pallas-addresses", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "pallas-primitives", "paste", "serde", @@ -2118,13 +2116,12 @@ dependencies = [ [[package]] name = "pallas-txbuilder" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "hex", "pallas-addresses", - "pallas-codec 0.22.0", - "pallas-crypto 0.22.0", + "pallas-codec 0.23.0", + "pallas-crypto 0.23.0", "pallas-primitives", "pallas-traverse", "pallas-wallet", @@ -2135,25 +2132,23 @@ dependencies = [ [[package]] name = "pallas-utxorpc" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ - "pallas-codec 0.22.0", + "pallas-codec 0.23.0", "pallas-primitives", "pallas-traverse", - "utxorpc", + "utxorpc-spec", ] [[package]] name = "pallas-wallet" -version = "0.22.0" -source = "git+https://github.com/txpipe/pallas.git?rev=691b6059e25b08bbe54cf8122d812bf37dd492e1#691b6059e25b08bbe54cf8122d812bf37dd492e1" +version = "0.23.0" dependencies = [ "bech32 0.9.1", "bip39", "cryptoxide", "ed25519-bip32", - "pallas-crypto 0.22.0", + "pallas-crypto 0.23.0", "rand", "thiserror", ] @@ -2187,7 +2182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdbb7b706f2afc610f3853550cdbbf6372fd324824a087806bd4480ea4996e24" dependencies = [ "heck", - "itertools", + "itertools 0.10.5", "prost", "prost-types", ] @@ -2342,7 +2337,7 @@ checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -2363,7 +2358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -3602,6 +3597,20 @@ dependencies = [ "utxorpc-spec-sync", ] +[[package]] +name = "utxorpc-spec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88f80e24bfe310d0972406d15c0892ff09b6c81ded2cdefc0183aac35cf0514f" +dependencies = [ + "bytes", + "pbjson", + "pbjson-types", + "prost", + "serde", + "tonic", +] + [[package]] name = "utxorpc-spec-cardano" version = "1.0.0-alpha.1" diff --git a/Cargo.toml b/Cargo.toml index 1f9907d0..91c1165d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,8 @@ authors = ["Santiago Carmuega "] [dependencies] -pallas = { git = "https://github.com/txpipe/pallas.git", rev = "691b6059e25b08bbe54cf8122d812bf37dd492e1" , features = ["unstable"] } -# pallas = { path = "../pallas/pallas", features = ["unstable"] } +# pallas = { version = "^0.23", features = ["unstable"] } +pallas = { path = "../pallas/pallas", features = ["unstable"] } gasket = { version = "^0.5", features = ["derive"] } # gasket = { path = "../../construkts/gasket-rs/gasket", features = ["derive"] } diff --git a/src/serve/grpc/sync.rs b/src/serve/grpc/sync.rs index 691e26d7..00f6f6c0 100644 --- a/src/serve/grpc/sync.rs +++ b/src/serve/grpc/sync.rs @@ -3,7 +3,7 @@ use pallas::{ crypto::hash::Hash, storage::rolldb::{chain, wal, Error}, }; -use std::pin::Pin; +use std::{ops::Deref, pin::Pin}; use tokio_stream::StreamExt; use tonic::{Code, Request, Response, Status}; use utxorpc::proto::sync::v1::*; @@ -133,44 +133,13 @@ impl chain_sync_service_server::ChainSyncService for ChainSyncServiceImpl { ) -> Result, tonic::Status> { let request = request.into_inner(); - let has_intersect = !request.intersect.is_empty(); - - for intersect in request.intersect { - let slot = intersect.index; - let hash: [u8; 32] = intersect.hash.to_vec().try_into().unwrap(); - - // start a RollStream where the first entry is the intersect point (if found) - let s = wal::RollStream::start_from_point(self.wal.clone(), (slot, hash.into())).map( - |res| { - res.map(|log| roll_to_tip_response(log)) - .map_err(|e| match e { - Error::NotFound => Status::not_found("intersect not found"), - e => Status::internal(format!("kvtable error: {e:?}")), - }) - }, - ); - - let mut s = Box::pin(s); - - // check the first entry so see if intersect was successful - match s.next().await { - // successfully intersected, return iter after intersect - Some(Ok(_)) => return Ok(Response::new(s)), - // try next intersect if intersect not found - Some(Err(s)) if s.code() == Code::NotFound => continue, - // return error encountered while trying to create iterator - Some(Err(s)) => return Err(Status::internal(format!("FT2 {s:?}"))), - // unreachable? - None => return Err(Status::internal("FT3")), - } - } - - // intersects were provided but we couldn't intersect WAL using them - if has_intersect { - return Err(Status::not_found("could not find intersect")); - } + let intersect: Vec<_> = request + .intersect + .iter() + .map(|x| (x.index, bytes_to_hash(&x.hash))) + .collect(); - let s = wal::RollStream::start_after(self.wal.clone(), None) + let s = wal::RollStream::intersect(self.wal.clone(), intersect) .map(|log| Ok(roll_to_tip_response(log))); Ok(Response::new(Box::pin(s))) diff --git a/src/sync/roll.rs b/src/sync/roll.rs index 0d2de2e9..de61e7b1 100644 --- a/src/sync/roll.rs +++ b/src/sync/roll.rs @@ -108,25 +108,25 @@ impl Worker {} impl gasket::framework::Worker for Worker { async fn bootstrap(stage: &Stage) -> Result { let last_seq_chain = if let Some(cursor) = stage.cursor_chain { - Some( - stage - .store - .find_wal_seq(cursor) - .ok_or(Error::server("could not find chain cursor on WAL")) - .or_panic()?, - ) + stage + .store + .find_wal_seq(&[cursor]) + .or_panic()? + .ok_or(Error::server("could not find chain cursor on WAL")) + .or_panic()? + .into() } else { None }; let last_seq_ledger = if let Some(cursor) = stage.cursor_ledger { - Some( - stage - .store - .find_wal_seq(cursor) - .ok_or(Error::server("could not find ledger cursor on WAL")) - .or_panic()?, - ) + stage + .store + .find_wal_seq(&[cursor]) + .or_panic()? + .ok_or(Error::server("could not find chain cursor on WAL")) + .or_panic()? + .into() } else { None };