Compression integration (draft) [JOLT-191]

Cargo.toml

@@ -102,6 +102,7 @@ jolt-sdk = { workspace = true, features = ["host"] }
 jolt-core = { workspace = true, features = ["default"] }
 tracer = { workspace = true, features = ["default"] }
 common = { workspace = true, features = ["std"] }
+expect-test = "1.5.1"
 
 [profile.test]
 opt-level = 3
@@ -122,30 +123,32 @@ inherits = "release"
 debug = false
 
 [patch.crates-io]
-ark-bn254 = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" }
-ark-ff = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" }
-ark-ec = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" }
-ark-serialize = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" }
+ark-bn254 = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression" }
+ark-ff = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression" }
+ark-ec = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression" }
+ark-serialize = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression" }
 allocative = { git = "https://github.com/facebookexperimental/allocative", rev = "85b773d85d526d068ce94724ff7a7b81203fc95e" }
+jolt-optimizations = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression" }
+dory-pcs = { git = "https://github.com/JuI3s/dory", branch = "compression-integration" }
 
 [workspace.metadata.cargo-machete]
 ignored = ["jolt-sdk"]
 
 [workspace.dependencies]
 # Cryptography and Math
-ark-bn254 = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout", default-features = false }
-ark-grumpkin = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout", default-features = false }
-ark-ec = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout", default-features = false }
-ark-ff = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout", default-features = false }
-ark-serialize = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout", default-features = false, features = [
+ark-bn254 = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression", default-features = false }
+ark-grumpkin = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression", default-features = false }
+ark-ec = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression", default-features = false }
+ark-ff = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression", default-features = false }
+ark-serialize = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression", default-features = false, features = [
     "derive",
 ] }
 ark-serialize-derive = { version = "0.5.0", default-features = false }
 ark-std = { version = "0.5.0", default-features = false }
 sha3 = "0.10.8"
 blake2 = "0.10"
 blake3 = { version = "1.5.0" }
-jolt-optimizations = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" }
+jolt-optimizations = { git = "https://github.com/JuI3s/arkworks-algebra", branch = "compression" }
 dory = { package = "dory-pcs", version = "0.1.0", features = ["backends", "cache", "disk-persistence"] }
 
 # Core Utilities

book/src/usage/guests_hosts/guests.md

@@ -119,3 +119,11 @@ fn int_to_string(n: i32) -> String {
 ```
 
 Note that `jolt_print` and `jolt_println` support format strings. The printed strings are written to stdout during RISC-V emulation of the guest.
+
+## Optimization level
+
+By default, the guest program is compiled with optimization level 3 to attempt to minimize the number of RISC-V instructions executed during emulation.
+
+To change this optimization level, set the `JOLT_GUEST_OPT` environment variable to one of the allowed values: `0`, `1`, `2`, `3`, `s`, or `z`.
+
+A common choice is `z` which optimizes for code size, at the cost of a larger number of instructions executed.

jolt-core/Cargo.toml

@@ -115,6 +115,7 @@ pprof = { version = "0.15", features = [
 prost = { version = "0.14", optional = true }
 bytemuck = "1.23.2"
 bytemuck_derive = "1.10.1"
+expect-test = "1.5.1"
 
 [dev-dependencies]
 criterion.workspace = true

jolt-core/src/guest/prover.rs

@@ -4,7 +4,7 @@
 use crate::poly::commitment::commitment_scheme::StreamingCommitmentScheme;
 use crate::poly::commitment::dory::DoryCommitmentScheme;
 use crate::transcripts::Transcript;
-use crate::zkvm::proof_serialization::JoltProof;
+use crate::zkvm::proof_serialization::JoltUncompressedProof;
 use crate::zkvm::{prover::JoltProverPreprocessing, ProverDebugInfo};
 use common::jolt_device::MemoryLayout;
 use tracer::JoltDevice;
@@ -35,11 +35,11 @@
     output_bytes: &mut [u8],
     preprocessing: &JoltProverPreprocessing<F, PCS>,
 ) -> (
-    JoltProof<F, PCS, FS>,
+    JoltUncompressedProof<F, PCS, FS>,
     JoltDevice,
     Option<ProverDebugInfo<F, FS, PCS>>,
 ) {
-    use crate::zkvm::prover::JoltCpuProver;
+    use crate::zkvm::prover::{JoltCpuProver, JoltProofCompressionFlag};
 
     let prover = JoltCpuProver::gen_from_elf(
         preprocessing,

jolt-core/src/guest/verifier.rs

@@ -6,7 +6,7 @@ use crate::guest::program::Program;
 use crate::poly::commitment::dory::DoryCommitmentScheme;
 use crate::transcripts::Transcript;
 use crate::utils::errors::ProofVerifyError;
-use crate::zkvm::proof_serialization::JoltProof;
+use crate::zkvm::proof_serialization::JoltUncompressedProof;
 use crate::zkvm::prover::JoltProverPreprocessing;
 use crate::zkvm::verifier::JoltVerifier;
 use crate::zkvm::verifier::JoltVerifierPreprocessing;
@@ -37,7 +37,7 @@ pub fn verify<F: JoltField, PCS: StreamingCommitmentScheme<Field = F>, FS: Trans
     inputs_bytes: &[u8],
     trusted_advice_commitment: Option<<PCS as CommitmentScheme>::Commitment>,
     outputs_bytes: &[u8],
-    proof: JoltProof<F, PCS, FS>,
+    proof: JoltUncompressedProof<F, PCS, FS>,
     preprocessing: &JoltVerifierPreprocessing<F, PCS>,
 ) -> Result<(), ProofVerifyError> {
     use common::jolt_device::JoltDevice;

jolt-core/src/host/program.rs

@@ -120,12 +120,23 @@ impl Program {
                 rust_flags.push("strip=symbols".to_string());
             }
 
-            rust_flags.extend_from_slice(&[
-                "-C".to_string(),
-                "opt-level=z".to_string(),
-                "--cfg".to_string(),
-                "getrandom_backend=\"custom\"".to_string(),
-            ]);
+            // Check environment variable opt level
+            // 3 is default if not set
+            let opt_level = std::env::var("JOLT_GUEST_OPT").unwrap_or_else(|_| "3".to_string());
+            // validate opt level
+            rust_flags.push("-C".to_string());
+            match opt_level.as_str() {
+                "0" | "1" | "2" | "3" | "s" | "z" => {
+                    rust_flags.push(format!("opt-level={opt_level}").to_string());
+                }
+                _ => {
+                    panic!(
+                        "Invalid JOLT_GUEST_OPT value: {opt_level}. Allowed values are 0, 1, 2, 3, s, z",
+                    );
+                }
+            }
+            rust_flags.push("--cfg".to_string());
+            rust_flags.push("getrandom_backend=\"custom\"".to_string());
 
             let target_triple = if self.std {
                 "riscv64imac-jolt-zkvm-elf"

jolt-core/src/poly/commitment/commitment_scheme.rs

@@ -9,6 +9,85 @@ use crate::{
     utils::{errors::ProofVerifyError, small_scalar::SmallScalar},
 };
 
+pub trait CompressedCommitmentScheme: CommitmentScheme {
+    type CompressedCommitment: Default
+        + Debug
+        + Sync
+        + Send
+        + PartialEq
+        + CanonicalSerialize
+        + CanonicalDeserialize
+        + AppendToTranscript
+        + Clone;
+    type CompressedProof: Sync + Send + CanonicalSerialize + CanonicalDeserialize + Clone + Debug;
+
+    /// Commits to a multilinear polynomial using the provided setup, where the commitment is compressed.
+    ///
+    /// # Arguments
+    /// * `poly` - The multilinear polynomial to commit to
+    /// * `setup` - The prover setup for the commitment scheme
+    ///
+    /// # Returns
+    /// A tuple containing the compressed commitment to the polynomial and a hint that can be used
+    /// to optimize opening proof generation
+    fn commit_compressed(
+        poly: &MultilinearPolynomial<Self::Field>,
+        setup: &Self::ProverSetup,
+    ) -> (Self::CompressedCommitment, Self::OpeningProofHint);
+
+    /// Generates a compressed proof of evaluation for a polynomial at a specific point.
+    ///
+    /// # Arguments
+    /// * `setup` - The prover setup for the commitment scheme
+    /// * `poly` - The multilinear polynomial being proved
+    /// * `opening_point` - The point at which the polynomial is evaluated
+    /// * `hint` - An optional hint that helps optimize the proof generation.
+    ///   When `None`, implementations should compute the hint internally if needed.
+    /// * `transcript` - The transcript for Fiat-Shamir transformation
+    ///
+    /// # Returns
+    /// A proof of the polynomial evaluation at the specified point
+    fn prove_compressed<ProofTranscript: Transcript>(
+        setup: &Self::ProverSetup,
+        poly: &MultilinearPolynomial<Self::Field>,
+        opening_point: &[<Self::Field as JoltField>::Challenge],
+        hint: Option<Self::OpeningProofHint>,
+        transcript: &mut ProofTranscript,
+    ) -> Self::CompressedProof;
+
+    fn verify_compressed<ProofTranscript: Transcript>(
+        proof: &Self::CompressedProof,
+        setup: &Self::VerifierSetup,
+        transcript: &mut ProofTranscript,
+        opening_point: &[<Self::Field as JoltField>::Challenge],
+        opening: &Self::Field,
+        commitment: &Self::CompressedCommitment,
+    ) -> Result<(), ProofVerifyError>;
+
+    /// Homomorphically combines multiple commitments into a single commitment, computed as a
+    /// linear combination with the given coefficients.
+    ///
+    /// TODO:
+    fn combine_commitments_compressed<C: Borrow<Self::CompressedCommitment>>(
+        _commitments: &[C],
+        _coeffs: &[Self::Field],
+    ) -> Self::CompressedCommitment {
+        panic!("`combine_commitments_compressed` is not implemented for this compressed commitment scheme 
+        or it does not support homomorphic combination of commitments");
+    }
+}
+
+pub trait CompressedStreamingCommitmentScheme:
+    CompressedCommitmentScheme + StreamingCommitmentScheme
+{
+    /// Compute tier 2 commitment from accumulated tier 1 commitments, where the output commitment is compressed.
+    fn aggregate_chunks_compressed(
+        setup: &Self::ProverSetup,
+        onehot_k: Option<usize>,
+        tier1_commitments: &[Self::ChunkState],
+    ) -> (Self::CompressedCommitment, Self::OpeningProofHint);
+}
+
 pub trait CommitmentScheme: Clone + Sync + Send + 'static {
     type Field: JoltField + Sized;
     type ProverSetup: Clone + Sync + Send + Debug + CanonicalSerialize + CanonicalDeserialize;
@@ -22,6 +101,7 @@ pub trait CommitmentScheme: Clone + Sync + Send + 'static {
         + CanonicalDeserialize
         + AppendToTranscript
         + Clone;
+
     type Proof: Sync + Send + CanonicalSerialize + CanonicalDeserialize + Clone + Debug;
     type BatchedProof: Sync + Send + CanonicalSerialize + CanonicalDeserialize;
     /// A hint that helps the prover compute an opening proof. Typically some byproduct of