Sp1 prover for halo2 + sp1 chunks

Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "bin",
     "integration",
+    "prover",
 ]
 
 [workspace.package]
@@ -27,23 +28,85 @@ serde_json = "1.0"
 tokio = { version = "1.32", features = ["full"] }
 
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1"  }
-prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll"] }
+prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "fix/revm_in_test", default-features = false, features = ["parallel_syn", "scroll"] }
 integration = { path = "integration" }
+scroll-prover = { path = "prover" }
+
 
 [patch.crates-io]
 halo2curves = { git = "https://github.com/scroll-tech/halo2curves", branch = "v0.1.0" }
 ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
 ethers-providers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
 ethers-signers  = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
+ethers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
+ethers-etherscan = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
+gobuild = { git = "https://github.com/scroll-tech/gobuild.git" }
 #ethers-etherscan = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
 #ethers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
+# ============= sp1 patches ========================
+#tiny-keccak = { git = "https://github.com/sp1-patches/tiny-keccak", branch = "patch-v2.0.2" }
+#k256 = { git = "https://github.com/sp1-patches/signatures", package = "ecdsa", branch = "patch-ecdsa-v0.16.9" }
+#sp1-lib = { git = "https://github.com/scroll-tech/sp1.git", branch = "scroll-dev-1019" }
+# notice bn is not compatible with revm 2.x which snark-verifier depends on
+#bn = { package = "substrate-bn", git = "https://github.com/sp1-patches/bn", branch = "patch-v0.6.0" }
+#sha2 = { git = "https://github.com/sp1-patches/RustCrypto-hashes", branch = "patch-v0.10.8", package = "sha2" }
+#sha3 = { git = "https://github.com/sp1-patches/RustCrypto-hashes", branch = "patch-v0.10.8", package = "sha3" }
+# ========== sproll evm patches: patched add rkyv support & MSRV 1.77 ============
+ruint = { git = "https://github.com/scroll-tech/uint.git", branch = "v1.12.3" }
+
+alloy = { git = "https://github.com/scroll-tech/alloy.git", branch = "v0.5.4" }
+alloy-eips = { git = "https://github.com/scroll-tech/alloy.git", branch = "v0.5.4" }
+alloy-serde = { git = "https://github.com/scroll-tech/alloy.git", branch = "v0.5.4" }
+
+alloy-eip2930 = { git = "https://github.com/scroll-tech/alloy-eips", branch = "v0.3.2" }
+alloy-eip7702 = { git = "https://github.com/scroll-tech/alloy-eips", branch = "v0.3.2" }
+
+alloy-core = { git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.10" }
+alloy-primitives = { git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.10" }
+alloy-sol-types = {git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.10" }
+# ========== revm patches, unify revm usage in sbv / zkevm-circuit
+revm = { git = "https://github.com/scroll-tech//revm", branch = "scroll-evm-executor/feat/v49/sp1"}
+#revm-primitives = { git = "https://github.com/scroll-tech//revm", branch = "scroll-evm-executor/feat/v49/sp1" }
+
+
 [patch."https://github.com/privacy-scaling-explorations/halo2.git"]
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1"  }
 [patch."https://github.com/privacy-scaling-explorations/poseidon.git"]
 poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "main" }
 [patch."https://github.com/privacy-scaling-explorations/bls12_381"]
 bls12_381 = { git = "https://github.com/scroll-tech/bls12_381", branch = "feat/impl_scalar_field" }
 
+[patch."https://github.com/scroll-tech/revm"]
+revm = { git = "https://github.com/scroll-tech//revm", branch = "scroll-evm-executor/feat/v49/sp1" }
+revm-primitives = { git = "https://github.com/scroll-tech//revm", branch = "scroll-evm-executor/feat/v49/sp1" }
+revm-precompile = { git = "https://github.com/scroll-tech//revm", branch = "scroll-evm-executor/feat/v49/sp1" }
+
+# uncomment to test on upstream
+#[patch."https://github.com/scroll-tech/sp1.git"]
+#sp1-sdk = "3.0.0"
+#sp1-prover = "3.0.0"
+#sp1-helper = "3.0.0"
+#sp1-lib = "3.0.0"
+#sp1-zkvm = "3.0.0"
+#sp1-core-executor = "3.0.0"
+#[patch."https://github.com/scroll-tech/sp1.git"]
+#sp1-sdk = { git = "https://github.com/succinctlabs/sp1.git", tag = "v3.0.0" }
+#sp1-prover = { git = "https://github.com/succinctlabs/sp1.git", tag = "v3.0.0" }
+#sp1-helper = { git = "https://github.com/succinctlabs/sp1.git", tag = "v3.0.0" }
+#sp1-lib = { git = "https://github.com/succinctlabs/sp1.git", tag = "v3.0.0" }
+#sp1-zkvm = { git = "https://github.com/succinctlabs/sp1.git", tag = "v3.0.0" }
+#sp1-core-executor = { git = "https://github.com/succinctlabs/sp1.git", tag = "v3.0.0" }
+
+#[patch."https://github.com/scroll-tech/poseidon-bn254"]
+#poseidon-bn254 = { git = "https://github.com/scroll-tech//poseidon-bn254.git", branch = "feat/syscall-code" }
+#[patch."https://github.com/scroll-tech/bn254"]
+#bn254 = { git = "https://github.com/scroll-tech//bn254.git", branch = "feat/syscall-code" }
+
+#[patch."https://github.com/scroll-tech/sp1-intrinsics"]
+#sp1-intrinsics = { path = "../sp1-intrinsics" }
+
+
+
 [profile.test]
 opt-level = 3
 debug-assertions = true

Makefile

@@ -6,6 +6,10 @@ RUST_MIN_STACK ?= 16777216
 export RUST_MIN_STACK
 RUST_BACKTRACE=1
 export RUST_BACKTRACE
+VERIFY_VK=false
+export VERIFY_VK
+SHARD_SIZE ?= 524288
+export SHARD_SIZE
 
 help: ## Display this help screen
 	@grep -h \
@@ -50,6 +54,9 @@ test-bundle-prove:
 test-e2e-prove:
 	@SCROLL_PROVER_DUMP_YUL=true cargo test --release -p integration --test e2e_tests test_e2e_prove_verify -- --exact --nocapture
 
+test-e2e-prove-hybrid:
+	@SCROLL_PROVER_DUMP_YUL=true cargo test --release -p integration --test e2e_tests test_e2e_prove_verify_hybrid -- --exact --nocapture
+
 test-batch-bundle-prove:
 	@SCROLL_PROVER_DUMP_YUL=true cargo test --release -p integration --test e2e_tests test_batch_bundle_verify -- --exact --nocapture
 

bin/Cargo.toml

@@ -21,6 +21,7 @@ tokio.workspace = true
 
 prover.workspace = true
 integration.workspace = true
+scroll-prover.workspace = true
 
 [[bin]]
 name = "trace_prover"
@@ -30,6 +31,10 @@ path = "src/trace_prover.rs"
 name = "chain_prover"
 path = "src/chain_prover.rs"
 
+[[bin]]
+name = "sp1_prover"
+path = "src/sp1_prover.rs"
+
 [features]
 default = []
 inner-prove = ["prover/test"]

bin/src/sp1_prover.rs

@@ -0,0 +1,72 @@
+use clap::Parser;
+use integration::{prove::prove_and_verify_sp1_chunk, test_util::load_chunk_compatible};
+use scroll_prover::{init_env_and_log, Sp1Prover, ChunkProvingTask};
+use std::env;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    /// Get params dir path.
+    #[clap(short, long = "params", default_value = "params")]
+    params_path: String,
+    /// Get asserts dir path.
+    #[clap(short, long = "assets", default_value = "test_assets")]
+    assets_path: String,
+    /// Get BlockTrace from file or dir.
+    #[clap(
+        short,
+        long = "trace",
+        default_value = "tests/extra_traces/batch_sproll/5224657.json"
+    )]
+    trace_path: String,
+}
+
+const DEFAULT_PARAM : &str = 
+r#"{
+  "strategy": "Vertical",
+  "k": 24,
+  "num_advice_per_phase": [
+    3
+  ],
+  "num_fixed": 1,
+  "num_lookup_advice_per_phase": [
+    1
+  ],
+  "lookup_bits": 22
+}"#;
+
+fn main() {
+    // Layer config files are located in `./integration/configs`.
+    env::set_current_dir("./integration").unwrap();
+    let output_dir = init_env_and_log("sp1_prover");
+    log::info!("Initialized ENV and created output-dir {output_dir}");
+
+    let args = Args::parse();
+
+    let traces = load_chunk_compatible(&args.trace_path).1;
+    prover::eth_types::constants::set_scroll_block_constants_with_trace(&traces[0]);
+    let chunk = ChunkProvingTask::new(traces);
+    let params_map = prover::Prover::load_params_map(
+        &args.params_path,
+        &[
+            *prover::INNER_DEGREE,
+            *prover::LAYER1_DEGREE,
+            *prover::LAYER2_DEGREE,
+        ],
+    );
+    log::warn!("must set VERIFY_VK to false, we also enfore SHARD_SIZE");
+    env::set_var("VERIFY_VK", "false");
+    env::set_var("SHARD_SIZE", "524288");
+    env::set_var("BASE_CONFIG_PARAMS", DEFAULT_PARAM);
+    let mut prover = Sp1Prover::from_params_and_assets(&params_map, &args.assets_path);
+    log::info!("Constructed sp1 chunk prover");
+
+    prove_and_verify_sp1_chunk(
+        &params_map,
+        &output_dir,
+        chunk,
+        &mut prover,
+        Some("0"), // same with `make test-chunk-prove`, to load vk        
+    );
+    log::info!("sp1 prove done");
+}

bin/src/trace_prover.rs

@@ -1,6 +1,6 @@
 use clap::Parser;
 use integration::{prove::prove_and_verify_chunk, test_util::load_chunk};
-use prover::{init_env_and_log, ChunkProvingTask};
+use prover::{init_env_and_log, ChunkProver, ChunkProvingTask};
 use std::env;
 
 #[derive(Parser, Debug)]
@@ -34,12 +34,15 @@ fn main() {
     let chunk = ChunkProvingTask::new(traces);
     let params_map =
         prover::Prover::load_params_map(&args.params_path, &prover::CHUNK_PROVER_DEGREES);
+    let mut prover = ChunkProver::from_params_and_assets(&params_map, &args.assets_path);
+    log::info!("Constructed chunk prover");
     prove_and_verify_chunk(
-        chunk,
-        Some("0"), // same with `make test-chunk-prove`, to load vk
         &params_map,
-        &args.assets_path,
         &output_dir,
+        chunk,
+        &mut prover,
+        Some("0"), // same with `make test-chunk-prove`, to load vk
+        true,
     );
     log::info!("chunk prove done");
 }

integration/.gitignore

@@ -17,3 +17,4 @@ test_seed
 
 *output_*
 outputs/
+elf/

integration/Cargo.toml

@@ -21,7 +21,7 @@ snark-verifier = { git = "https://github.com/scroll-tech/snark-verifier", branch
 snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop", default-features = false, features = ["loader_halo2", "loader_evm", "halo2-pse"] }
 
 halo2_proofs.workspace = true
-prover.workspace = true
+prover = { path = "../prover", package = "scroll-prover"}
 
 [features]
 default = ["prove_verify"]

integration/release.sh

@@ -9,6 +9,8 @@ export OPT_MEM=true
 export RUST_MIN_STACK=100000000
 export PARAM_SEED=bb4b94a1bbef58c4b5fcda6c900629b5 
 
+
+
 function goerli() {
 		TRACE_DIR="0327-correct-deletion"
 		d="617365.json"

integration/src/prove.rs

@@ -1,35 +1,69 @@
 use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG};
 use prover::{
-    get_blob_bytes, BatchData, BatchProofV2, BatchProver, BatchProvingTask, BatchVerifier,
-    BundleProvingTask, ChunkInfo, ChunkProver, ChunkProvingTask, ChunkVerifier, MAX_AGG_SNARKS,
+    chunk_trace_to_witness_block, get_blob_bytes, BatchData, BatchProofV2, BatchProver,
+    BatchProvingTask, BatchVerifier, BundleProvingTask, ChunkInfo, ChunkProofV2,
+    ChunkProofV2Metadata, ChunkProver, ChunkProvingTask, ChunkVerifier, Snark, MAX_AGG_SNARKS,
+    Sp1Prover,
 };
 use std::{collections::BTreeMap, env, time::Instant};
 
-use crate::verifier::EVMVerifier;
+use crate::verifier::{new_chunk_verifier, EVMVerifier};
 
 /// The `output_dir` is assumed to output_dir of chunk proving.
 pub fn new_batch_prover<'a>(
     params_map: &'a BTreeMap<u32, ParamsKZG<Bn256>>,
     output_dir: &str,
 ) -> BatchProver<'a> {
-    env::set_var("HALO2_CHUNK_PROTOCOL", "protocol_chunk_0.protocol");
-    env::set_var("SP1_CHUNK_PROTOCOL", "protocol_chunk_0.protocol");
+    env::set_var("HALO2_CHUNK_PROTOCOL", "protocol_chunk_halo2.protocol");
+    env::set_var("SP1_CHUNK_PROTOCOL", "protocol_chunk_sp1.protocol");
+    env::set_var("SCROLL_PROVER_ASSETS_DIR", output_dir);
     let prover = BatchProver::from_params_and_assets(params_map, output_dir);
     log::info!("Constructed batch prover");
 
     prover
 }
 
-pub fn prove_and_verify_chunk(
+/// prove_and_verify_sp1_chunk would expect a sp1 snark name "sp1_snark_<chunk_id>.json"
+pub fn prove_and_verify_sp1_chunk(
+    params_map: &BTreeMap<u32, ParamsKZG<Bn256>>,
+    output_dir: &str,
     chunk: ChunkProvingTask,
+    prover: &mut Sp1Prover,
     chunk_identifier: Option<&str>,
+) -> ChunkProofV2 {
+    let chunk_identifier =
+        chunk_identifier.map_or_else(|| chunk.identifier(), |name| name.to_string());
+
+    let now = Instant::now();
+    let chunk_proof = prover
+        .gen_chunk_proof(chunk, Some(&chunk_identifier), None, Some(output_dir))
+        .expect("cannot generate chunk snark");
+    log::info!(
+        "finish generating chunk snark, elapsed: {:?}",
+        now.elapsed()
+    );
+
+    // output_dir is used to load chunk vk
+    env::set_var(
+        "CHUNK_VK_FILENAME",
+        &format!("vk_chunk_{chunk_identifier}.vkey"),
+    );
+    let verifier = new_chunk_verifier(params_map, output_dir);
+    let snark = Snark::try_from(&chunk_proof).expect("should be ok");
+    assert!(verifier.verify_snark(snark));
+    log::info!("Verified sp1 chunk proof");
+
+    chunk_proof
+}
+
+pub fn prove_and_verify_chunk(
     params_map: &BTreeMap<u32, ParamsKZG<Bn256>>,
-    assets_path: &str,
     output_dir: &str,
-) {
-    let mut prover = ChunkProver::from_params_and_assets(params_map, assets_path);
-    log::info!("Constructed chunk prover");
-
+    chunk: ChunkProvingTask,
+    prover: &mut ChunkProver,
+    chunk_identifier: Option<&str>,
+    skip_verify: bool,
+) -> ChunkProofV2 {
     let chunk_identifier =
         chunk_identifier.map_or_else(|| chunk.identifier(), |name| name.to_string());
 
@@ -42,6 +76,13 @@ pub fn prove_and_verify_chunk(
         now.elapsed()
     );
 
+    // there is an issue: if snark is restore from disk, the pk is not generated
+    // and the dumping process of proof would write the existed vk with 0 bytes
+    // and cause verify failed
+    // the work-around is skip verify in e2e test
+    if skip_verify {
+        return chunk_proof;
+    }
     // output_dir is used to load chunk vk
     env::set_var(
         "CHUNK_VK_FILENAME",
@@ -52,6 +93,8 @@ pub fn prove_and_verify_chunk(
         .verify_chunk_proof(&chunk_proof)
         .expect("should verify");
     log::info!("Verified chunk proof");
+
+    chunk_proof
 }
 
 pub fn prove_and_verify_batch(