Merge pull request 0xPolygonMiden#784 from 0xPolygonMiden/vlopes11-re…

…move-merkle-set

feat: add miden-crypto merkle store as advice provider backend

assembly/src/assembler/instruction/crypto_ops.rs

@@ -145,7 +145,7 @@ pub(super) fn mtree_set(span: &mut SpanBuilder) -> Result<Option<CodeBlock>, Ass
     // stack: [d, i, R_old, V_new, ...]
 
     // stack: [V_old, R_new, ...] (29 cycles)
-    update_mtree(span, false)
+    update_mtree(span, true)
 }
 
 /// Appends the MRUPDATE op with a parameter of "true" and stack manipulations to the span block as

core/src/lib.rs

@@ -14,7 +14,7 @@ pub use ::crypto::{Word, ONE, WORD_SIZE, ZERO};
 pub mod crypto {
     pub mod merkle {
         pub use ::crypto::merkle::{
-            MerkleError, MerklePath, MerklePathSet, MerkleTree, NodeIndex, SimpleSmt,
+            MerkleError, MerklePath, MerklePathSet, MerkleStore, MerkleTree, NodeIndex, SimpleSmt,
         };
     }
 

docs/src/intro/overview.md

@@ -43,6 +43,6 @@ The *advice provider* component is responsible for supplying nondeterministic in
 The advice provider consists of three components:
 * **Advice stack** which is a one-dimensional array of field elements. Being a stack, the VM can either push new elements onto the advice stack, or pop the elements from its top.
 * **Advice map** which is a key-value map where keys are words and values are vectors of field elements. The VM can copy values from the advice map onto the advice stack as well as insert new values into the advice map (e.g., from a region of memory).
-* **Merkle sets** which contain structured data reducible to Merkle paths. Some examples of merkle sets are: Merkle tree, Sparse Merkle tree, a collection of Merkle paths. Every merkle set can be uniquely identified by its root. The VM can request Merkle paths from an merkle set, as well as update an merkle set by modifying one of its nodes (this will also change the root of the modified merkle set).
+* **Merkle store** which contain structured data reducible to Merkle paths. Some examples of such structures are: Merkle tree, Sparse Merkle tree, and a collection of Merkle paths. The VM can request Merkle paths from the Merkle store, as well as mutate it by updating or merging nodes contained in the store.
 
 The prover initializes the advice provider prior to executing a program, and from that point on the advice provider is manipulated solely by executing operations on the VM.

miden/src/lib.rs

@@ -12,6 +12,6 @@ pub use processor::{
 };
 pub use prover::{
     math, prove, Digest, ExecutionProof, FieldExtension, HashFunction, InputError, MerkleError,
-    MerkleSet, Program, ProofOptions, StackOutputs, StarkProof, Word,
+    Program, ProofOptions, StackOutputs, StarkProof, Word,
 };
 pub use verifier::{verify, VerificationError};

miden/tests/integration/air/chiplets/hasher.rs

@@ -1,8 +1,10 @@
 use crate::build_op_test;
-use crate::helpers::crypto::{init_merkle_leaf, init_merkle_leaves};
-use processor::MerkleSet;
+use crate::helpers::crypto::{init_merkle_leaf, init_merkle_store};
 use rand_utils::rand_vector;
-use vm_core::StarkField;
+use vm_core::{
+    crypto::merkle::{MerkleStore, MerkleTree},
+    StarkField, Word,
+};
 
 #[test]
 fn hperm() {
@@ -25,8 +27,8 @@ fn mtree_get() {
     let asm_op = "mtree_get";
 
     let index = 3usize;
-    let leaves = init_merkle_leaves(&[1, 2, 3, 4, 5, 6, 7, 8]);
-    let tree = MerkleSet::new_merkle_tree(leaves).unwrap();
+    let (leaves, store) = init_merkle_store(&[1, 2, 3, 4, 5, 6, 7, 8]);
+    let tree = MerkleTree::new(leaves.clone()).unwrap();
 
     let stack_inputs = [
         tree.root()[0].as_int(),
@@ -37,35 +39,35 @@ fn mtree_get() {
         tree.depth() as u64,
     ];
 
-    build_op_test!(asm_op, &stack_inputs, &[], vec![tree])
+    build_op_test!(asm_op, &stack_inputs, &[], store)
         .prove_and_verify(stack_inputs.to_vec(), false);
 }
 
 #[test]
 fn mtree_set() {
     let asm_op = "mtree_set";
-    let (stack_inputs, tree) = build_mtree_update_test_inputs();
+    let (stack_inputs, store, _leaves) = build_mtree_update_test_inputs();
 
-    build_op_test!(asm_op, &stack_inputs, &[], vec![tree])
+    build_op_test!(asm_op, &stack_inputs, &[], store)
         .prove_and_verify(stack_inputs.to_vec(), false);
 }
 
 #[test]
 fn mtree_cwm() {
     let asm_op = "mtree_cwm";
-    let (stack_inputs, tree) = build_mtree_update_test_inputs();
+    let (stack_inputs, store, _leaves) = build_mtree_update_test_inputs();
 
-    build_op_test!(asm_op, &stack_inputs, &[], vec![tree]).prove_and_verify(stack_inputs, false);
+    build_op_test!(asm_op, &stack_inputs, &[], store).prove_and_verify(stack_inputs, false);
 }
 
 /// Helper function that builds a test stack and Merkle tree for testing mtree updates.
-fn build_mtree_update_test_inputs() -> (Vec<u64>, MerkleSet) {
+fn build_mtree_update_test_inputs() -> (Vec<u64>, MerkleStore, Vec<Word>) {
     let index = 5_usize;
-    let leaves = init_merkle_leaves(&[1, 2, 3, 4, 5, 6, 7, 8]);
-    let tree = MerkleSet::new_merkle_tree(leaves.clone()).unwrap();
+    let (leaves, store) = init_merkle_store(&[1, 2, 3, 4, 5, 6, 7, 8]);
+    let tree = MerkleTree::new(leaves.clone()).unwrap();
 
     let new_node = init_merkle_leaf(9);
-    let mut new_leaves = leaves;
+    let mut new_leaves = leaves.clone();
     new_leaves[index] = new_node;
 
     let stack_inputs = vec![
@@ -81,5 +83,5 @@ fn build_mtree_update_test_inputs() -> (Vec<u64>, MerkleSet) {
         tree.depth() as u64,
     ];
 
-    (stack_inputs, tree)
+    (stack_inputs, store, leaves)
 }

miden/tests/integration/helpers/crypto.rs

@@ -1,9 +1,15 @@
 use super::Felt;
-use vm_core::{FieldElement, Word};
+use vm_core::{crypto::merkle::MerkleStore, FieldElement, Word};
 
 // CRYPTO HELPER FUNCTIONS
 // ================================================================================================
 
+pub fn init_merkle_store(values: &[u64]) -> (Vec<Word>, MerkleStore) {
+    let leaves = init_merkle_leaves(values);
+    let store = MerkleStore::new().with_merkle_tree(leaves.clone()).unwrap();
+    (leaves, store)
+}
+
 pub fn init_merkle_leaves(values: &[u64]) -> Vec<Word> {
     values.iter().map(|&v| init_merkle_leaf(v)).collect()
 }

miden/tests/integration/helpers/mod.rs

@@ -3,7 +3,9 @@ pub use processor::{AdviceInputs, StackInputs};
 use processor::{ExecutionError, ExecutionTrace, Process, VmStateIterator};
 use proptest::prelude::*;
 use stdlib::StdLibrary;
-pub use vm_core::{stack::STACK_TOP_SIZE, Felt, FieldElement, Program, StackOutputs};
+pub use vm_core::{
+    crypto::merkle::MerkleStore, stack::STACK_TOP_SIZE, Felt, FieldElement, Program, StackOutputs,
+};
 
 pub mod crypto;
 

miden/tests/integration/main.rs

@@ -34,9 +34,9 @@ fn multi_output_program() {
 /// * `stack_inputs` (optional): the initial inputs which must be at the top of the stack before
 /// executing the `source`. Stack inputs can be provided independently without any advice inputs.
 /// * `advice_stack` (optional): the initial advice stack values. When provided, `stack_inputs` and
-/// `merkle_sets` are also expected.
-/// * `merkle_sets` (optional): the initial merkle set values. When provided, `stack_inputs` and
-/// `advice_tape` are also expected.
+/// `merkle_store` are also expected.
+/// * `merkle_store` (optional): the initial merkle set values. When provided, `stack_inputs` and
+/// `advice_stack` are also expected.
 #[macro_export]
 macro_rules! build_op_test {
     ($op_str:expr) => {{
@@ -59,9 +59,9 @@ macro_rules! build_op_test {
 /// * `stack_inputs` (optional): the initial inputs which must be at the top of the stack before
 /// executing the `source`. Stack inputs can be provided independently without any advice inputs.
 /// * `advice_stack` (optional): the initial advice stack values. When provided, `stack_inputs` and
-/// `merkle_sets` are also expected.
-/// * `merkle_sets` (optional): the initial merkle set values. When provided, `stack_inputs` and
-/// `advice_tape` are also expected.
+/// `merkle_store` are also expected.
+/// * `merkle_store` (optional): the initial merkle set values. When provided, `stack_inputs` and
+/// `advice_stack` are also expected.
 #[macro_export]
 macro_rules! build_test {
     ($($params:tt)+) => {{
@@ -79,9 +79,9 @@ macro_rules! build_test {
 /// * `stack_inputs` (optional): the initial inputs which must be at the top of the stack before
 /// executing the `source`. Stack inputs can be provided independently without any advice inputs.
 /// * `advice_stack` (optional): the initial advice stack values. When provided, `stack_inputs` and
-/// `merkle_sets` are also expected.
-/// * `merkle_sets` (optional): the initial merkle set values. When provided, `stack_inputs` and
-/// `advice_tape` are also expected.
+/// `merkle_store` are also expected.
+/// * `merkle_store` (optional): the initial merkle set values. When provided, `stack_inputs` and
+/// `advice_stack` are also expected.
 #[macro_export]
 macro_rules! build_debug_test {
     ($($params:tt)+) => {{
@@ -117,16 +117,16 @@ macro_rules! build_test_by_mode {
         }
     }};
     (
-        $in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr, $merkle_sets:expr
+        $in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr
     ) => {{
         let stack_inputs: Vec<u64> = $stack_inputs.to_vec();
         let stack_inputs = $crate::helpers::StackInputs::try_from_values(stack_inputs).unwrap();
         let stack_values: Vec<u64> = $advice_stack.to_vec();
+        let store = $crate::helpers::MerkleStore::new();
         let advice_inputs = $crate::helpers::AdviceInputs::default()
             .with_stack_values(stack_values)
             .unwrap()
-            .with_merkle_sets($merkle_sets)
-            .unwrap();
+            .with_merkle_store(store);
 
         $crate::helpers::Test {
             source: String::from($source),
@@ -136,15 +136,33 @@ macro_rules! build_test_by_mode {
             in_debug_mode: $in_debug_mode,
         }
     }};
-    ($in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr, $advice_sets:expr, $advice_map:expr) => {{
+    (
+        $in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr, $advice_merkle_store:expr
+    ) => {{
         let stack_inputs: Vec<u64> = $stack_inputs.to_vec();
         let stack_inputs = $crate::helpers::StackInputs::try_from_values(stack_inputs).unwrap();
         let stack_values: Vec<u64> = $advice_stack.to_vec();
         let advice_inputs = $crate::helpers::AdviceInputs::default()
             .with_stack_values(stack_values)
             .unwrap()
-            .with_merkle_sets($advice_sets)
+            .with_merkle_store($advice_merkle_store);
+
+        $crate::helpers::Test {
+            source: String::from($source),
+            kernel: None,
+            stack_inputs,
+            advice_inputs,
+            in_debug_mode: $in_debug_mode,
+        }
+    }};
+    ($in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr, $advice_merkle_store:expr, $advice_map:expr) => {{
+        let stack_inputs: Vec<u64> = $stack_inputs.to_vec();
+        let stack_inputs = $crate::helpers::StackInputs::try_from_values(stack_inputs).unwrap();
+        let stack_values: Vec<u64> = $advice_stack.to_vec();
+        let advice_inputs = $crate::helpers::AdviceInputs::default()
+            .with_stack_values(stack_values)
             .unwrap()
+            .with_merkle_store($advice_merkle_store)
             .with_map($advice_map);
 
         $crate::helpers::Test {

miden/tests/integration/operations/crypto_ops.rs

@@ -1,13 +1,12 @@
-use processor::MerkleSet;
 use rand_utils::rand_vector;
 use vm_core::{
     chiplets::hasher::{apply_permutation, hash_elements, STATE_WIDTH},
-    crypto::merkle::NodeIndex,
+    crypto::merkle::{MerkleTree, NodeIndex},
     Felt, FieldElement, StarkField,
 };
 
 use crate::build_op_test;
-use crate::helpers::crypto::{init_merkle_leaf, init_merkle_leaves};
+use crate::helpers::crypto::{init_merkle_leaf, init_merkle_store};
 
 // TESTS
 // ================================================================================================
@@ -105,8 +104,8 @@ fn mtree_get() {
     let asm_op = "mtree_get";
 
     let index = 3usize;
-    let leaves = init_merkle_leaves(&[1, 2, 3, 4, 5, 6, 7, 8]);
-    let tree = MerkleSet::new_merkle_tree(leaves.clone()).unwrap();
+    let (leaves, store) = init_merkle_store(&[1, 2, 3, 4, 5, 6, 7, 8]);
+    let tree = MerkleTree::new(leaves.clone()).unwrap();
 
     let stack_inputs = [
         tree.root()[0].as_int(),
@@ -128,20 +127,20 @@ fn mtree_get() {
         tree.root()[0].as_int(),
     ];
 
-    let test = build_op_test!(asm_op, &stack_inputs, &[], vec![tree]);
+    let test = build_op_test!(asm_op, &stack_inputs, &[], store);
     test.expect_stack(&final_stack);
 }
 
 #[test]
 fn mtree_update() {
     let index = 5usize;
-    let leaves = init_merkle_leaves(&[1, 2, 3, 4, 5, 6, 7, 8]);
-    let tree = MerkleSet::new_merkle_tree(leaves.clone()).unwrap();
+    let (leaves, store) = init_merkle_store(&[1, 2, 3, 4, 5, 6, 7, 8]);
+    let tree = MerkleTree::new(leaves.clone()).unwrap();
 
     let new_node = init_merkle_leaf(9);
-    let mut new_leaves = leaves;
+    let mut new_leaves = leaves.clone();
     new_leaves[index] = new_node;
-    let new_tree = MerkleSet::new_merkle_tree(new_leaves).unwrap();
+    let new_tree = MerkleTree::new(new_leaves).unwrap();
 
     let stack_inputs = [
         new_node[0].as_int(),
@@ -176,7 +175,7 @@ fn mtree_update() {
         new_tree.root()[0].as_int(),
     ];
 
-    let test = build_op_test!(asm_op, &stack_inputs, &[], vec![tree.clone()]);
+    let test = build_op_test!(asm_op, &stack_inputs, &[], store.clone());
     test.expect_stack(&final_stack);
 
     // --- mtree_cwm ----------------------------------------------------------------------
@@ -195,7 +194,7 @@ fn mtree_update() {
         new_tree.root()[0].as_int(),
     ];
 
-    let test = build_op_test!(asm_op, &stack_inputs, &[], vec![tree]);
+    let test = build_op_test!(asm_op, &stack_inputs, &[], store);
     test.expect_stack(&final_stack);
 }
 

miden/tests/integration/operations/io_ops/adv_ops.rs

@@ -14,7 +14,7 @@ fn adv_push() {
         final_stack.copy_from_slice(&advice_stack[..n]);
         final_stack.reverse();
 
-        let test = build_op_test!(source, &[], &advice_stack, vec![]);
+        let test = build_op_test!(source, &[], &advice_stack);
         test.expect_stack(&final_stack);
     };
 
@@ -42,7 +42,7 @@ fn adv_loadw() {
     let mut final_stack = advice_stack;
     final_stack.reverse();
 
-    let test = build_op_test!(asm_op, &[8, 7, 6, 5], &advice_stack, vec![]);
+    let test = build_op_test!(asm_op, &[8, 7, 6, 5], &advice_stack);
     test.expect_stack(&final_stack);
 }
 
@@ -82,6 +82,6 @@ fn adv_pipe() {
     final_stack.reverse();
     final_stack.push(2);
 
-    let test = build_test!(source, &[], &advice_stack, vec![]);
+    let test = build_test!(source, &[], &advice_stack);
     test.expect_stack(&final_stack);
 }

miden/tests/integration/operations/io_ops/mod.rs

@@ -47,13 +47,13 @@ fn mem_stream_pipe() {
     // initialize with anything other than zeros, since the stack is set to 0s between the adv_pipe
     // and mem_stream operations in the source script.
     let stack_inputs = [1, 1, 1, 1, 1, 1, 1, 1];
-    let test = build_test!(source, &stack_inputs, &advice_stack, vec![]);
+    let test = build_test!(source, &stack_inputs, &advice_stack);
     let final_stack = test.get_last_stack_state();
     assert_eq!(final_stack[0..4], final_stack[4..8]);
 
     // --- the same stack values should yield the same results from adv_pipe and mem_stream -------
     // initialize with all zeros, just like between the adv_pipe and mem_stream operations above.
-    let test = build_test!(source, &[], &advice_stack, vec![]);
+    let test = build_test!(source, &[], &advice_stack);
     let final_stack = test.get_last_stack_state();
     assert_eq!(final_stack[0..4], final_stack[4..8]);
 

miden/tests/integration/stdlib/crypto/fri/channel.rs

@@ -1,5 +1,4 @@
-use miden::MerkleSet;
-use vm_core::{Felt, FieldElement};
+use vm_core::{crypto::merkle::MerklePathSet, Felt, FieldElement};
 use winter_fri::{FriProof, VerifierError};
 
 use winterfell::{
@@ -13,7 +12,7 @@ pub trait UnBatch<E: FieldElement, H: ElementHasher> {
         positions: &[usize],
         domain_size: usize,
         layer_commitments: Vec<<H as HasherTrait>::Digest>,
-    ) -> (Vec<MerkleSet>, Vec<([u8; 32], Vec<Felt>)>);
+    ) -> (Vec<MerklePathSet>, Vec<([u8; 32], Vec<Felt>)>);
 }
 
 pub struct MidenFriVerifierChannel<E: FieldElement, H: ElementHasher<BaseField = E::BaseField>> {