Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/hybrid snark agg #1438

Draft
wants to merge 25 commits into
base: v0.13
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
7ec5588
pin rust to 1.77 (#1429)
lispc Oct 9, 2024
252b8d9
use revm v3.5.0 instead of snark-verifier sdk
roynalnaruto Oct 9, 2024
5b3d535
export deploy_and_call
roynalnaruto Oct 9, 2024
6dd70e7
feat: add chunk kind
roynalnaruto Oct 16, 2024
21b4093
initial work | to be tested
roynalnaruto Oct 17, 2024
73efa6e
fix: borrow ctx from loader only once
roynalnaruto Oct 17, 2024
dbe3d93
more logs
roynalnaruto Oct 17, 2024
756f4af
read from protocol json files
roynalnaruto Oct 18, 2024
e7b778e
dir + name
roynalnaruto Oct 18, 2024
7bb4d86
reference already duplicated somewhere in
roynalnaruto Oct 18, 2024
0ad5448
minor updates, bump snark-verifier
roynalnaruto Oct 18, 2024
dc52147
refactor
roynalnaruto Oct 18, 2024
b621bce
clean up + refactor
roynalnaruto Oct 22, 2024
1beeb35
chunk prover error and refactor
roynalnaruto Oct 23, 2024
667ace4
layer-2 compress only (sp1 chunk proof)
roynalnaruto Oct 23, 2024
a8c2733
more refactoring and better inline docs
roynalnaruto Oct 24, 2024
d684c06
refactor prover::aggregator module
roynalnaruto Oct 24, 2024
b849391
return BatchProverError instead of using anyhow
roynalnaruto Oct 24, 2024
c8cd343
tidy up lib.rs of prover crate
roynalnaruto Oct 25, 2024
fc050c4
refactor prover::types and minor changes
roynalnaruto Oct 25, 2024
65b7c7c
more refactoring
roynalnaruto Oct 25, 2024
f8fdd32
move more modules into prover::utils
roynalnaruto Oct 25, 2024
80f2e52
proof v2
roynalnaruto Oct 26, 2024
44595ac
use bundle proof v2
roynalnaruto Oct 26, 2024
3aaf9c4
fix(soundness): init_state depends on halo2/sp1 route
roynalnaruto Nov 4, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chunk prover error and refactor
roynalnaruto committed Oct 23, 2024
commit 1beeb3573f01621385b36ea2de55a4f11c6eefa9
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions prover/Cargo.toml
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@ serde.workspace = true
serde_derive = "1.0"
serde_json = { workspace = true, features = ["unbounded_depth"] }
serde_stacker.workspace = true
thiserror = "1.0"
sha2 ="0.10.2"
revm = { version = "3.5.0", default-features = false, features = ["std"] }

6 changes: 5 additions & 1 deletion prover/src/aggregator/prover.rs
Original file line number Diff line number Diff line change
@@ -345,7 +345,11 @@ pub fn check_chunk_hashes(
) -> Result<()> {
for (idx, (in_arg, chunk_proof)) in chunk_hashes_proofs.iter().enumerate() {
let in_proof = &chunk_proof.chunk_info;
crate::proof::compare_chunk_info(&format!("{name} chunk num {idx}"), in_arg, in_proof)?;
if let Err(e) =
crate::proof::compare_chunk_info(&format!("{name} chunk num {idx}"), in_arg, in_proof)
{
bail!(e);
}
}
Ok(())
}
4 changes: 2 additions & 2 deletions prover/src/inner/prover.rs
Original file line number Diff line number Diff line change
@@ -2,8 +2,8 @@ use crate::{
common,
config::INNER_DEGREE,
io::serialize_vk,
utils::{chunk_trace_to_witness_block, gen_rng},
zkevm::circuit::TargetCircuit,
utils::gen_rng,
zkevm::circuit::{chunk_trace_to_witness_block, TargetCircuit},
Proof,
};
use anyhow::Result;
18 changes: 9 additions & 9 deletions prover/src/proof/chunk.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use super::{dump_as_json, dump_data, dump_vk, from_json_file, Proof};
use crate::{types::base64, zkevm::SubCircuitRowUsage};
use aggregator::ChunkInfo;
use anyhow::{bail, Result};
use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::ProvingKey};
use serde_derive::{Deserialize, Serialize};
use snark_verifier::Protocol;
@@ -37,33 +36,34 @@ pub struct ChunkProof {
macro_rules! compare_field {
($desc:expr, $field:ident, $lhs:ident, $rhs:ident) => {
if $lhs.$field != $rhs.$field {
bail!(
return Err(format!(
"{} chunk different {}: {} != {}",
$desc,
stringify!($field),
$lhs.$field,
$rhs.$field
);
));
}
};
}

/// Check chunk info is consistent with chunk info embedded inside proof
pub fn compare_chunk_info(name: &str, lhs: &ChunkInfo, rhs: &ChunkInfo) -> Result<()> {
pub fn compare_chunk_info(name: &str, lhs: &ChunkInfo, rhs: &ChunkInfo) -> Result<(), String> {
compare_field!(name, chain_id, lhs, rhs);
compare_field!(name, prev_state_root, lhs, rhs);
compare_field!(name, post_state_root, lhs, rhs);
compare_field!(name, withdraw_root, lhs, rhs);
compare_field!(name, data_hash, lhs, rhs);
if lhs.tx_bytes != rhs.tx_bytes {
bail!(
return Err(format!(
"{} chunk different {}: {} != {}",
name,
"tx_bytes",
hex::encode(&lhs.tx_bytes),
hex::encode(&rhs.tx_bytes)
);
));
}

Ok(())
}

@@ -74,7 +74,7 @@ impl ChunkProof {
chunk_info: ChunkInfo,
chunk_kind: ChunkKind,
row_usages: Vec<SubCircuitRowUsage>,
) -> Result<Self> {
) -> anyhow::Result<Self> {
let protocol = serde_json::to_vec(&snark.protocol)?;
let proof = Proof::new(snark.proof, &snark.instances, pk);

@@ -87,11 +87,11 @@ impl ChunkProof {
})
}

pub fn from_json_file(dir: &str, name: &str) -> Result<Self> {
pub fn from_json_file(dir: &str, name: &str) -> anyhow::Result<Self> {
from_json_file(dir, &dump_filename(name))
}

pub fn dump(&self, dir: &str, name: &str) -> Result<()> {
pub fn dump(&self, dir: &str, name: &str) -> anyhow::Result<()> {
let filename = dump_filename(name);

// Dump vk and protocol.
13 changes: 1 addition & 12 deletions prover/src/utils.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
#![allow(deprecated)]
use crate::{
types::BlockTraceJsonRpcResult,
zkevm::circuit::{block_traces_to_witness_block, print_chunk_stats},
};
use crate::types::BlockTraceJsonRpcResult;
use anyhow::{bail, Result};
use chrono::Utc;
use eth_types::l2_types::BlockTrace;
@@ -127,14 +124,6 @@ pub fn metric_of_witness_block(block: &Block) -> ChunkMetric {
}
}

pub fn chunk_trace_to_witness_block(chunk_trace: Vec<BlockTrace>) -> Result<Block> {
if chunk_trace.is_empty() {
bail!("Empty chunk trace");
}
print_chunk_stats(&chunk_trace);
block_traces_to_witness_block(chunk_trace)
}

// Return the output dir.
pub fn init_env_and_log(id: &str) -> String {
dotenvy::dotenv().ok();
2 changes: 2 additions & 0 deletions prover/src/zkevm.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
#[cfg(feature = "scroll")]
mod capacity_checker;
pub mod circuit;
mod error;
mod prover;
mod verifier;

pub use self::prover::Prover;
#[cfg(feature = "scroll")]
pub use capacity_checker::{CircuitCapacityChecker, RowUsage};
pub use error::ChunkProverError;
use serde::{Deserialize, Serialize};
pub use verifier::Verifier;

5 changes: 1 addition & 4 deletions prover/src/zkevm/capacity_checker.rs
Original file line number Diff line number Diff line change
@@ -137,10 +137,7 @@ impl CircuitCapacityChecker {
self.acc_row_usage.clone()
}
}
pub fn estimate_circuit_capacity(
&mut self,
trace: BlockTrace,
) -> Result<RowUsage, anyhow::Error> {
pub fn estimate_circuit_capacity(&mut self, trace: BlockTrace) -> anyhow::Result<RowUsage> {
let (mut estimate_builder, codedb_prev) =
if let Some((code_db, sdb, mpt_state)) = self.builder_ctx.take() {
// here we create a new builder for another (sealed) witness block
7 changes: 3 additions & 4 deletions prover/src/zkevm/circuit.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
use builder::dummy_witness_block;
use halo2_proofs::halo2curves::bn256::Fr;
use snark_verifier_sdk::CircuitExt;
use zkevm_circuits::{super_circuit::params::ScrollSuperCircuit, util::SubCircuit, witness};

mod builder;
pub use self::builder::{
block_traces_to_witness_block, calculate_row_usage_of_witness_block, finalize_builder,
print_chunk_stats,
block_traces_to_witness_block, calculate_row_usage_of_witness_block,
chunk_trace_to_witness_block, finalize_builder,
};

pub use zkevm_circuits::super_circuit::params::{MAX_CALLDATA, MAX_INNER_BLOCKS, MAX_TXS};
@@ -23,7 +22,7 @@ pub trait TargetCircuit {
where
Self: Sized,
{
let witness_block = dummy_witness_block()?;
let witness_block = builder::dummy_witness_block()?;
let circuit = Self::from_witness_block(&witness_block)?;
Ok(circuit)
}
79 changes: 47 additions & 32 deletions prover/src/zkevm/circuit/builder.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use crate::zkevm::SubCircuitRowUsage;
use anyhow::{bail, Result};
use bus_mapping::circuit_input_builder::CircuitInputBuilder;
use crate::zkevm::{ChunkProverError, SubCircuitRowUsage};
use bus_mapping::{circuit_input_builder::CircuitInputBuilder, Error as CircuitBuilderError};
use eth_types::{l2_types::BlockTrace, ToWord};
use itertools::Itertools;
use mpt_zktrie::state::ZkTrieHash;
@@ -12,12 +11,14 @@ use zkevm_circuits::{

pub fn calculate_row_usage_of_witness_block(
witness_block: &Block,
) -> Result<Vec<SubCircuitRowUsage>> {
) -> Result<Vec<SubCircuitRowUsage>, ChunkProverError> {
let rows = ScrollSuperCircuit::min_num_rows_block_subcircuits(witness_block);

// Check whether we need to "estimate" poseidon sub circuit row usage
if witness_block.mpt_updates.smt_traces.is_empty() {
bail!("light mode no longer supported");
return Err(ChunkProverError::Custom(
"light mode no longer supported".to_string(),
));
}

let first_block_num = witness_block.first_block_number();
@@ -34,58 +35,44 @@ pub fn calculate_row_usage_of_witness_block(
.sum::<usize>(),
rows,
);
let row_usage_details: Vec<SubCircuitRowUsage> = rows

Ok(rows
.into_iter()
.map(|x| SubCircuitRowUsage {
name: x.name,
row_number: x.row_num_real,
})
.collect_vec();
Ok(row_usage_details)
}

pub fn print_chunk_stats(block_traces: &[BlockTrace]) {
let num_blocks = block_traces.len();
let num_txs = block_traces
.iter()
.map(|b| b.transactions.len())
.sum::<usize>();
let total_tx_len = block_traces
.iter()
.flat_map(|b| b.transactions.iter().map(|t| t.data.len()))
.sum::<usize>();
log::info!(
"check capacity of block traces, num_block {}, num_tx {}, tx total len {}",
num_blocks,
num_txs,
total_tx_len
);
.collect_vec())
}

pub fn dummy_witness_block() -> Result<Block> {
pub fn dummy_witness_block() -> anyhow::Result<Block> {
log::debug!("generate dummy witness block");
let dummy_chain_id = 0;
let witness_block = zkevm_circuits::witness::dummy_witness_block(dummy_chain_id);
log::debug!("generate dummy witness block done");
Ok(witness_block)
}

pub fn block_traces_to_witness_block(block_traces: Vec<BlockTrace>) -> Result<Block> {
pub fn block_traces_to_witness_block(
block_traces: Vec<BlockTrace>,
) -> Result<Block, ChunkProverError> {
if block_traces.is_empty() {
bail!("use dummy_witness_block instead");
return Err(ChunkProverError::Custom(
"empty block traces! hint: use dummy_witness_block instead".to_string(),
));
}
let block_num = block_traces.len();
let total_tx_num = block_traces
.iter()
.map(|b| b.transactions.len())
.sum::<usize>();
if total_tx_num > MAX_TXS {
bail!(
return Err(ChunkProverError::Custom(format!(
"tx num overflow {}, block range {} to {}",
total_tx_num,
block_traces[0].header.number.unwrap(),
block_traces[block_num - 1].header.number.unwrap()
);
)));
}
log::info!(
"block_traces_to_witness_block, block num {}, tx num {}",
@@ -116,8 +103,18 @@ pub fn block_traces_to_witness_block(block_traces: Vec<BlockTrace>) -> Result<Bl
Ok(witness_block)
}

pub fn chunk_trace_to_witness_block(
chunk_trace: Vec<BlockTrace>,
) -> Result<Block, ChunkProverError> {
if chunk_trace.is_empty() {
return Err(ChunkProverError::Custom("Empty chunk trace".to_string()));
}
print_chunk_stats(&chunk_trace);
block_traces_to_witness_block(chunk_trace)
}

/// Finalize building and return witness block
pub fn finalize_builder(builder: &mut CircuitInputBuilder) -> Result<Block> {
pub fn finalize_builder(builder: &mut CircuitInputBuilder) -> Result<Block, CircuitBuilderError> {
builder.finalize_building()?;

log::debug!("converting builder.block to witness block");
@@ -152,3 +149,21 @@ pub fn finalize_builder(builder: &mut CircuitInputBuilder) -> Result<Block> {

Ok(witness_block)
}

fn print_chunk_stats(block_traces: &[BlockTrace]) {
let num_blocks = block_traces.len();
let num_txs = block_traces
.iter()
.map(|b| b.transactions.len())
.sum::<usize>();
let total_tx_len = block_traces
.iter()
.flat_map(|b| b.transactions.iter().map(|t| t.data.len()))
.sum::<usize>();
log::info!(
"check capacity of block traces, num_block {}, num_tx {}, tx total len {}",
num_blocks,
num_txs,
total_tx_len
);
}
29 changes: 29 additions & 0 deletions prover/src/zkevm/error.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/// Various errors potentially encountered during proof generation.
#[derive(thiserror::Error, Debug)]
pub enum ChunkProverError {
/// Indicates that the halo2-based [`SuperCircuit`][super_circ] does not have sufficient
/// capacity to populate block traces from all the blocks in the chunk. The error encapsulates
/// the [`RowUsage`][row_usage] observed from populating the chunk.
///
/// [super_circ]: zkevm_circuits::super_circuit::SuperCircuit
/// [row_usage]: crate::zkevm::RowUsage
#[error("halo2 circuit-capacity exceeded")]
CircuitCapacityOverflow(crate::zkevm::RowUsage),
/// Represents an error propagated from the [`bus_mapping`] crate.
#[error(transparent)]
CircuitBuilder(#[from] bus_mapping::Error),
/// Represents the [`halo2 error`][halo2_error] being propagated.
///
/// [halo2_error]: halo2_proofs::plonk::Error
#[error(transparent)]
Halo2(#[from] halo2_proofs::plonk::Error),
/// Represents all other custom errors.
#[error("custom error: {0}")]
Custom(String),
}

impl From<String> for ChunkProverError {
fn from(value: String) -> Self {
Self::Custom(value)
}
}
103 changes: 56 additions & 47 deletions prover/src/zkevm/prover.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
use std::collections::BTreeMap;

use crate::{
common, config::LayerId, consts::CHUNK_VK_FILENAME, io::try_to_read, proof::compare_chunk_info,
types::ChunkProvingTask, utils::chunk_trace_to_witness_block,
zkevm::circuit::calculate_row_usage_of_witness_block, ChunkProof,
common,
config::LayerId,
consts::CHUNK_VK_FILENAME,
io::try_to_read,
proof::compare_chunk_info,
types::ChunkProvingTask,
zkevm::{
circuit::{calculate_row_usage_of_witness_block, chunk_trace_to_witness_block},
ChunkProverError, RowUsage,
},
ChunkProof,
};
use aggregator::ChunkInfo;
use anyhow::Result;
use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG};

#[derive(Debug)]
@@ -67,62 +74,64 @@ impl<'params> Prover<'params> {
pub fn gen_chunk_proof(
&mut self,
chunk: ChunkProvingTask,
chunk_identifier: Option<&str>,
chunk_id: Option<&str>,
inner_id: Option<&str>,
output_dir: Option<&str>,
) -> Result<ChunkProof> {
) -> Result<ChunkProof, ChunkProverError> {
assert!(!chunk.is_empty());

let chunk_identifier =
chunk_identifier.map_or_else(|| chunk.identifier(), |name| name.to_string());

let chunk_proof = match output_dir
.and_then(|output_dir| ChunkProof::from_json_file(output_dir, &chunk_identifier).ok())
{
Some(proof) => Ok(proof),
None => {
let witness_block = chunk_trace_to_witness_block(chunk.block_traces)?;
let row_usage = calculate_row_usage_of_witness_block(&witness_block)?;
log::info!("Got witness block");

let chunk_info = ChunkInfo::from_witness_block(&witness_block, false);
if let Some(chunk_info_input) = chunk.chunk_info.as_ref() {
compare_chunk_info(
&format!("gen_chunk_proof {chunk_identifier:?}"),
&chunk_info,
chunk_info_input,
)?;
}
let snark = self.prover_impl.load_or_gen_final_chunk_snark(
&chunk_identifier,
&witness_block,
inner_id,
output_dir,
)?;
let chunk_id = chunk_id.map_or_else(|| chunk.identifier(), |name| name.to_string());

self.check_vk();
let cached_proof =
output_dir.and_then(|dir| ChunkProof::from_json_file(dir, &chunk_id).ok());

let result = ChunkProof::new(
snark,
self.prover_impl.pk(LayerId::Layer2.id()),
chunk_info,
chunk.chunk_kind,
row_usage,
);
let chunk_proof = cached_proof.unwrap_or({
let witness_block = chunk_trace_to_witness_block(chunk.block_traces)?;
log::info!("Got witness block");

if let (Some(output_dir), Ok(proof)) = (output_dir, &result) {
proof.dump(output_dir, &chunk_identifier)?;
}
let sub_circuit_row_usages = calculate_row_usage_of_witness_block(&witness_block)?;
let row_usage = RowUsage::from_row_usage_details(sub_circuit_row_usages.clone());
if !row_usage.is_ok {
return Err(ChunkProverError::CircuitCapacityOverflow(row_usage));
}

result
let chunk_info = ChunkInfo::from_witness_block(&witness_block, false);
if let Some(chunk_info_input) = chunk.chunk_info.as_ref() {
compare_chunk_info(
&format!("gen_chunk_proof {chunk_id:?}"),
&chunk_info,
chunk_info_input,
)?;
}
}?;
let snark = self
.prover_impl
.load_or_gen_final_chunk_snark(&chunk_id, &witness_block, inner_id, output_dir)
.map_err(|e| ChunkProverError::Custom(e.to_string()))?;

self.check_vk();

let result = ChunkProof::new(
snark,
self.prover_impl.pk(LayerId::Layer2.id()),
chunk_info,
chunk.chunk_kind,
sub_circuit_row_usages,
);

if let (Some(output_dir), Ok(proof)) = (output_dir, &result) {
proof
.dump(output_dir, &chunk_id)
.map_err(|e| ChunkProverError::Custom(e.to_string()))?;
}

result.map_err(|e| ChunkProverError::Custom(e.to_string()))?
});

if let Some(verifier) = &self.verifier {
if !verifier.verify_chunk_proof(chunk_proof.clone()) {
anyhow::bail!("chunk prover cannot generate valid proof");
return Err(String::from("chunk proof verification failed").into());
}
log::info!("verify_chunk_proof done");
log::info!("chunk proof verified OK");
}

Ok(chunk_proof)