From ec81769ddf66ce6e206c2f7b49738862b4442807 Mon Sep 17 00:00:00 2001 From: Michal Nazarewicz Date: Fri, 26 Nov 2021 21:04:21 +0100 Subject: [PATCH] chain: remove nested `tests` modules (#5486) After moving tests from `/tests` to `/src/tests` and thus bringing them all into a single executable with a single namespace we ended up with aesthetically unpleasing nested paths such as: tests::catching_up::tests::test_chunk_grieving Flatten the structure by removing the inner `tests` module. Issue: https://github.com/near/nearcore/issues/5371 --- chain/chain/src/tests/doomslug.rs | 569 ++++--- chain/chain/src/tests/gc.rs | 918 ++++++----- chain/chain/src/tests/mod.rs | 1 + chain/client/src/tests/catching_up.rs | 1827 +++++++++++----------- chain/client/src/tests/consensus.rs | 507 +++--- chain/client/src/tests/cross_shard_tx.rs | 900 ++++++----- chain/client/src/tests/mod.rs | 2 + nightly/expensive.txt | 112 +- 8 files changed, 2392 insertions(+), 2444 deletions(-) diff --git a/chain/chain/src/tests/doomslug.rs b/chain/chain/src/tests/doomslug.rs index bcce16dda64..6036b1e1c7c 100644 --- a/chain/chain/src/tests/doomslug.rs +++ b/chain/chain/src/tests/doomslug.rs @@ -1,334 +1,325 @@ -#[cfg(test)] -#[cfg(feature = "expensive_tests")] -mod tests { - use near_primitives::time::Clock; - use rand::{thread_rng, Rng}; - use std::collections::{HashMap, HashSet}; - use std::sync::Arc; - use std::time::{Duration, Instant}; - - use crate::{Doomslug, DoomslugThresholdMode}; - use near_crypto::{KeyType, SecretKey}; - use near_primitives::block::Approval; - use near_primitives::hash::{hash, CryptoHash}; - use near_primitives::types::{ApprovalStake, BlockHeight}; - use near_primitives::validator_signer::InMemoryValidatorSigner; - - fn block_hash(height: BlockHeight, ord: usize) -> CryptoHash { - hash(([height.to_le_bytes(), ord.to_le_bytes()].concat()).as_ref()) - } - - fn get_msg_delivery_time(now: Instant, gst: Instant, delta: Duration) -> Instant { - std::cmp::max(now, gst) - + Duration::from_millis(thread_rng().gen_range(0, delta.as_millis()) as u64) - } - - /// Runs a single iteration of a fuzz test given specific time until global stabilization and - /// the max delay on messages. - /// Returns amount of time it took to produce a doomslug final block at height 50, as well as the - /// largest height encountered - /// - /// # Arguments - /// * `time_to_gst` - number of milliseconds before global stabilization time - /// * `delta` - max message delay - /// * `height_goal` - the appearance of a block at this (or higher) height with finality - /// will end the test - fn one_iter( - time_to_gst: Duration, - delta: Duration, - height_goal: BlockHeight, - ) -> (Duration, BlockHeight) { - let account_ids = - vec!["test1", "test2", "test3", "test4", "test5", "test6", "test7", "test8"]; - let stakes = account_ids - .iter() - .map(|account_id| ApprovalStake { - account_id: account_id.parse().unwrap(), - stake_this_epoch: 1, - stake_next_epoch: 1, - public_key: SecretKey::from_seed(KeyType::ED25519, account_id).public_key(), - }) - .map(|stake| (stake, false)) - .collect::>(); - let signers = account_ids - .iter() - .map(|account_id| { - Arc::new(InMemoryValidatorSigner::from_seed( - account_id.parse().unwrap(), - KeyType::ED25519, - account_id, - )) - }) - .collect::>(); - let mut doomslugs = signers - .iter() - .map(|signer| { - Doomslug::new( - 0, - Duration::from_millis(200), - Duration::from_millis(1000), - Duration::from_millis(100), - delta * 20, // some arbitrary number larger than delta * 6 - Some(signer.clone()), - DoomslugThresholdMode::TwoThirds, - ) - }) - .collect::>(); - - let mut now = Clock::instant(); - let started = now; - - let gst = now + time_to_gst; - let mut approval_queue: Vec<(Approval, Instant)> = vec![]; - let mut block_queue: Vec<(BlockHeight, usize, BlockHeight, Instant, CryptoHash)> = vec![]; - let mut largest_produced_height: BlockHeight = 1; - let mut chain_lengths = HashMap::new(); - let mut hash_to_block_info: HashMap = - HashMap::new(); - let mut hash_to_prev_hash: HashMap = HashMap::new(); - - let mut blocks_with_finality: Vec<(CryptoHash, BlockHeight)> = vec![]; - - chain_lengths.insert(block_hash(1, 0), 1); +use near_primitives::time::Clock; +use rand::{thread_rng, Rng}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::{Doomslug, DoomslugThresholdMode}; +use near_crypto::{KeyType, SecretKey}; +use near_primitives::block::Approval; +use near_primitives::hash::{hash, CryptoHash}; +use near_primitives::types::{ApprovalStake, BlockHeight}; +use near_primitives::validator_signer::InMemoryValidatorSigner; + +fn block_hash(height: BlockHeight, ord: usize) -> CryptoHash { + hash(([height.to_le_bytes(), ord.to_le_bytes()].concat()).as_ref()) +} - for ds in doomslugs.iter_mut() { - ds.set_tip(now, block_hash(1, 0), 1, 1); - hash_to_block_info.insert(block_hash(1, 0), (1, 1, block_hash(1, 0))); - } +fn get_msg_delivery_time(now: Instant, gst: Instant, delta: Duration) -> Instant { + std::cmp::max(now, gst) + + Duration::from_millis(thread_rng().gen_range(0, delta.as_millis()) as u64) +} - let mut is_done = false; - while !is_done { - now = now + Duration::from_millis(25); - let mut new_approval_queue = vec![]; - let mut new_block_queue = vec![]; - - // 1. Process approvals - for approval in approval_queue.into_iter() { - if approval.1 > now { - new_approval_queue.push(approval); - } else { - let me = (approval.0.target_height % 8) as usize; - - // Make test1 and test2 be offline and never send approvals - if matches!(approval.0.account_id.as_ref(), "test1" | "test2") { - continue; - } +/// Runs a single iteration of a fuzz test given specific time until global stabilization and +/// the max delay on messages. +/// Returns amount of time it took to produce a doomslug final block at height 50, as well as the +/// largest height encountered +/// +/// # Arguments +/// * `time_to_gst` - number of milliseconds before global stabilization time +/// * `delta` - max message delay +/// * `height_goal` - the appearance of a block at this (or higher) height with finality +/// will end the test +fn one_iter( + time_to_gst: Duration, + delta: Duration, + height_goal: BlockHeight, +) -> (Duration, BlockHeight) { + let account_ids = vec!["test1", "test2", "test3", "test4", "test5", "test6", "test7", "test8"]; + let stakes = account_ids + .iter() + .map(|account_id| ApprovalStake { + account_id: account_id.parse().unwrap(), + stake_this_epoch: 1, + stake_next_epoch: 1, + public_key: SecretKey::from_seed(KeyType::ED25519, account_id).public_key(), + }) + .map(|stake| (stake, false)) + .collect::>(); + let signers = account_ids + .iter() + .map(|account_id| { + Arc::new(InMemoryValidatorSigner::from_seed( + account_id.parse().unwrap(), + KeyType::ED25519, + account_id, + )) + }) + .collect::>(); + let mut doomslugs = signers + .iter() + .map(|signer| { + Doomslug::new( + 0, + Duration::from_millis(200), + Duration::from_millis(1000), + Duration::from_millis(100), + delta * 20, // some arbitrary number larger than delta * 6 + Some(signer.clone()), + DoomslugThresholdMode::TwoThirds, + ) + }) + .collect::>(); + + let mut now = Clock::instant(); + let started = now; + + let gst = now + time_to_gst; + let mut approval_queue: Vec<(Approval, Instant)> = vec![]; + let mut block_queue: Vec<(BlockHeight, usize, BlockHeight, Instant, CryptoHash)> = vec![]; + let mut largest_produced_height: BlockHeight = 1; + let mut chain_lengths = HashMap::new(); + let mut hash_to_block_info: HashMap = + HashMap::new(); + let mut hash_to_prev_hash: HashMap = HashMap::new(); + + let mut blocks_with_finality: Vec<(CryptoHash, BlockHeight)> = vec![]; + + chain_lengths.insert(block_hash(1, 0), 1); + + for ds in doomslugs.iter_mut() { + ds.set_tip(now, block_hash(1, 0), 1, 1); + hash_to_block_info.insert(block_hash(1, 0), (1, 1, block_hash(1, 0))); + } - // Generally make 20% of the remaining approvals to drop - if thread_rng().gen_range(0, 10) < 2 { - continue; - } + let mut is_done = false; + while !is_done { + now = now + Duration::from_millis(25); + let mut new_approval_queue = vec![]; + let mut new_block_queue = vec![]; + + // 1. Process approvals + for approval in approval_queue.into_iter() { + if approval.1 > now { + new_approval_queue.push(approval); + } else { + let me = (approval.0.target_height % 8) as usize; + + // Make test1 and test2 be offline and never send approvals + if matches!(approval.0.account_id.as_ref(), "test1" | "test2") { + continue; + } - doomslugs[me].on_approval_message(now, &approval.0, &stakes); + // Generally make 20% of the remaining approvals to drop + if thread_rng().gen_range(0, 10) < 2 { + continue; } + + doomslugs[me].on_approval_message(now, &approval.0, &stakes); } - approval_queue = new_approval_queue; - - // 2. Process blocks - for block in block_queue.into_iter() { - if block.3 > now { - new_block_queue.push(block); - } else { - let ds = &mut doomslugs[block.1 as usize]; - if block.0 as BlockHeight > ds.get_tip().1 { - // Accept all the blocks from the tip till this block - let mut block_infos = vec![(block.0, block.2, block.4)]; - for block_index in 0..50 { - if block_index == 49 { - assert!(false); - } + } + approval_queue = new_approval_queue; + + // 2. Process blocks + for block in block_queue.into_iter() { + if block.3 > now { + new_block_queue.push(block); + } else { + let ds = &mut doomslugs[block.1 as usize]; + if block.0 as BlockHeight > ds.get_tip().1 { + // Accept all the blocks from the tip till this block + let mut block_infos = vec![(block.0, block.2, block.4)]; + for block_index in 0..50 { + if block_index == 49 { + assert!(false); + } - let last_block = block_infos.last().unwrap(); - let prev_block_info = hash_to_block_info - .get(&hash_to_prev_hash.get(&last_block.2).unwrap()) - .unwrap(); + let last_block = block_infos.last().unwrap(); + let prev_block_info = hash_to_block_info + .get(&hash_to_prev_hash.get(&last_block.2).unwrap()) + .unwrap(); - if prev_block_info.0 as BlockHeight <= ds.get_tip().1 { - break; - } - block_infos.push(*prev_block_info); + if prev_block_info.0 as BlockHeight <= ds.get_tip().1 { + break; } + block_infos.push(*prev_block_info); + } - for block_info in block_infos.into_iter().rev() { - if block_info.0 > ds.get_tip().1 { - ds.set_tip( - now, - block_info.2, - block_info.0 as BlockHeight, - block_info.1, - ); - } + for block_info in block_infos.into_iter().rev() { + if block_info.0 > ds.get_tip().1 { + ds.set_tip( + now, + block_info.2, + block_info.0 as BlockHeight, + block_info.1, + ); } } } } - block_queue = new_block_queue; + } + block_queue = new_block_queue; - // 3. Process timers - for ds in doomslugs.iter_mut() { - for approval in ds.process_timer(now) { - approval_queue.push((approval, get_msg_delivery_time(now, gst, delta))); - } + // 3. Process timers + for ds in doomslugs.iter_mut() { + for approval in ds.process_timer(now) { + approval_queue.push((approval, get_msg_delivery_time(now, gst, delta))); } + } - // 4. Produce blocks - 'outer: for (bp_ord, ds) in doomslugs.iter_mut().enumerate() { - for target_height in - (ds.get_tip().1 + 1)..=ds.get_largest_height_crossing_threshold() - { - if ds.ready_to_produce_block(now, target_height, true) { - let num_blocks_to_produce = if bp_ord < 3 { 2 } else { 1 }; - - for block_ord in 0..num_blocks_to_produce { - let parent_hash = ds.get_tip().0; - - let prev_height = hash_to_block_info.get(&parent_hash).unwrap().0; - let prev_prev_height = if prev_height <= 1 { - 0 - } else { - let prev_prev_hash = hash_to_prev_hash.get(&parent_hash).unwrap(); - hash_to_block_info.get(&prev_prev_hash).unwrap().0 - }; - - let is_final = target_height == prev_height + 1 - && prev_height == prev_prev_height + 1; - - let last_final_height = if is_final { - target_height - 2 - } else { - hash_to_block_info.get(&parent_hash).unwrap().1 - }; - - if target_height >= 2048 { - println!("Largest produced_height: {}", largest_produced_height); - for ds in doomslugs.iter() { - println!( - " - tip: ({:?}), final_height: {}, timer height: {}", - ds.get_tip(), - ds.get_largest_final_height(), - ds.get_timer_height() - ); - } - assert!(false); - break 'outer; - } - let block_hash = block_hash(target_height, block_ord); - for whom in 0..8 { - let block_info = ( - target_height, - whom, - last_final_height, - get_msg_delivery_time(now, gst, delta), - block_hash, - ); - block_queue.push(block_info); - } - - hash_to_block_info - .insert(block_hash, (target_height, last_final_height, block_hash)); - hash_to_prev_hash.insert(block_hash, parent_hash); + // 4. Produce blocks + 'outer: for (bp_ord, ds) in doomslugs.iter_mut().enumerate() { + for target_height in (ds.get_tip().1 + 1)..=ds.get_largest_height_crossing_threshold() { + if ds.ready_to_produce_block(now, target_height, true) { + let num_blocks_to_produce = if bp_ord < 3 { 2 } else { 1 }; - assert!(chain_lengths.get(&block_hash).is_none()); - let prev_length = *chain_lengths.get(&ds.get_tip().0).unwrap(); - chain_lengths.insert(block_hash, prev_length + 1); + for block_ord in 0..num_blocks_to_produce { + let parent_hash = ds.get_tip().0; - if is_final && target_height != 2 { - blocks_with_finality.push(( - hash_to_prev_hash.get(&parent_hash).unwrap().clone(), - target_height - 2, - )); - } + let prev_height = hash_to_block_info.get(&parent_hash).unwrap().0; + let prev_prev_height = if prev_height <= 1 { + 0 + } else { + let prev_prev_hash = hash_to_prev_hash.get(&parent_hash).unwrap(); + hash_to_block_info.get(&prev_prev_hash).unwrap().0 + }; - if target_height > largest_produced_height { - largest_produced_height = target_height; - } - if target_height >= height_goal && is_final { - assert!(prev_length + 1 > 20); // make sure we actually built some chain - is_done = true; - } + let is_final = + target_height == prev_height + 1 && prev_height == prev_prev_height + 1; - // Accept our own block (only accept the last if are maliciously producing multiple, - // so that `ds.get_tip(...)` doesn't return the new block on the next iteration) - if block_ord + 1 == num_blocks_to_produce { - ds.set_tip( - now, - block_hash, - target_height as BlockHeight, - last_final_height, + let last_final_height = if is_final { + target_height - 2 + } else { + hash_to_block_info.get(&parent_hash).unwrap().1 + }; + + if target_height >= 2048 { + println!("Largest produced_height: {}", largest_produced_height); + for ds in doomslugs.iter() { + println!( + " - tip: ({:?}), final_height: {}, timer height: {}", + ds.get_tip(), + ds.get_largest_final_height(), + ds.get_timer_height() ); } + assert!(false); + break 'outer; + } + let block_hash = block_hash(target_height, block_ord); + for whom in 0..8 { + let block_info = ( + target_height, + whom, + last_final_height, + get_msg_delivery_time(now, gst, delta), + block_hash, + ); + block_queue.push(block_info); } - } - } - } - // 5. In the liveness proof we rely on timers always being within delta from each other - // Validate that assumption - for i in 0..8 { - for j in (i + 1)..8 { - let ith_timer_start = doomslugs[i].get_timer_start(); - let jth_timer_start = doomslugs[j].get_timer_start(); - - // Only makes sense for timers that are more than delta in the past, since for more - // recent timers the other participant's start time might be in the future - if now - ith_timer_start >= delta && now - jth_timer_start >= delta { - if ith_timer_start > jth_timer_start { - assert!(ith_timer_start - jth_timer_start <= delta); - } else { - assert!(jth_timer_start - ith_timer_start <= delta); + hash_to_block_info + .insert(block_hash, (target_height, last_final_height, block_hash)); + hash_to_prev_hash.insert(block_hash, parent_hash); + + assert!(chain_lengths.get(&block_hash).is_none()); + let prev_length = *chain_lengths.get(&ds.get_tip().0).unwrap(); + chain_lengths.insert(block_hash, prev_length + 1); + + if is_final && target_height != 2 { + blocks_with_finality.push(( + hash_to_prev_hash.get(&parent_hash).unwrap().clone(), + target_height - 2, + )); + } + + if target_height > largest_produced_height { + largest_produced_height = target_height; + } + if target_height >= height_goal && is_final { + assert!(prev_length + 1 > 20); // make sure we actually built some chain + is_done = true; + } + + // Accept our own block (only accept the last if are maliciously producing multiple, + // so that `ds.get_tip(...)` doesn't return the new block on the next iteration) + if block_ord + 1 == num_blocks_to_produce { + ds.set_tip( + now, + block_hash, + target_height as BlockHeight, + last_final_height, + ); } } } } } - // We successfully got to the `height_goal`. Check that all the blocks are building only on - // doomslug final blocks - for (block_hash, (block_height, _, _)) in hash_to_block_info.iter() { - let mut seen_hashes = HashSet::new(); - let mut block_hash = block_hash.clone(); - seen_hashes.insert(block_hash); - - loop { - match hash_to_prev_hash.get(&block_hash) { - None => break, - Some(prev_block_hash) => { - block_hash = *prev_block_hash; - seen_hashes.insert(block_hash); + // 5. In the liveness proof we rely on timers always being within delta from each other + // Validate that assumption + for i in 0..8 { + for j in (i + 1)..8 { + let ith_timer_start = doomslugs[i].get_timer_start(); + let jth_timer_start = doomslugs[j].get_timer_start(); + + // Only makes sense for timers that are more than delta in the past, since for more + // recent timers the other participant's start time might be in the future + if now - ith_timer_start >= delta && now - jth_timer_start >= delta { + if ith_timer_start > jth_timer_start { + assert!(ith_timer_start - jth_timer_start <= delta); + } else { + assert!(jth_timer_start - ith_timer_start <= delta); } } } + } + } - for (block_hash, height) in blocks_with_finality.iter() { - assert!(*height >= *block_height || seen_hashes.contains(block_hash)); + // We successfully got to the `height_goal`. Check that all the blocks are building only on + // doomslug final blocks + for (block_hash, (block_height, _, _)) in hash_to_block_info.iter() { + let mut seen_hashes = HashSet::new(); + let mut block_hash = block_hash.clone(); + seen_hashes.insert(block_hash); + + loop { + match hash_to_prev_hash.get(&block_hash) { + None => break, + Some(prev_block_hash) => { + block_hash = *prev_block_hash; + seen_hashes.insert(block_hash); + } } } - (now - started, largest_produced_height) + for (block_hash, height) in blocks_with_finality.iter() { + assert!(*height >= *block_height || seen_hashes.contains(block_hash)); + } } - #[test] - fn test_fuzzy_doomslug_liveness_and_safety() { - for (time_to_gst_millis, height_goal) in - &[(0, 200), (1000, 200), (10000, 300), (100000, 400), (500000, 500)] - { - for delta in &[100, 300, 500, 1000, 2000, 4000] { + (now - started, largest_produced_height) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_fuzzy_doomslug_liveness_and_safety() { + for (time_to_gst_millis, height_goal) in + &[(0, 200), (1000, 200), (10000, 300), (100000, 400), (500000, 500)] + { + for delta in &[100, 300, 500, 1000, 2000, 4000] { + println!("Staring set of tests. Time to GST: {}, delta: {}", time_to_gst_millis, delta); + for _iter in 0..10 { + let (took, height) = one_iter( + Duration::from_millis(*time_to_gst_millis), + Duration::from_millis(*delta), + *height_goal, + ); println!( - "Staring set of tests. Time to GST: {}, delta: {}", - time_to_gst_millis, delta + " --> Took {} (simulated) milliseconds and {} heights", + took.as_millis(), + height ); - for _iter in 0..10 { - let (took, height) = one_iter( - Duration::from_millis(*time_to_gst_millis), - Duration::from_millis(*delta), - *height_goal, - ); - println!( - " --> Took {} (simulated) milliseconds and {} heights", - took.as_millis(), - height - ); - } } } } diff --git a/chain/chain/src/tests/gc.rs b/chain/chain/src/tests/gc.rs index 1e0a1ef3b57..5cc5d43b500 100644 --- a/chain/chain/src/tests/gc.rs +++ b/chain/chain/src/tests/gc.rs @@ -1,475 +1,509 @@ -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use crate::chain::Chain; - use crate::test_utils::KeyValueRuntime; - use crate::types::{ChainGenesis, Tip}; - use crate::DoomslugThresholdMode; - use near_crypto::KeyType; - use near_primitives::block::Block; - use near_primitives::merkle::PartialMerkleTree; - use near_primitives::shard_layout::ShardUId; - use near_primitives::types::{NumBlocks, NumShards, StateRoot}; - use near_primitives::validator_signer::InMemoryValidatorSigner; - use near_store::test_utils::{create_test_store, gen_changes}; - use near_store::{ShardTries, StoreUpdate, Trie, WrappedTrieChanges}; - use rand::Rng; - - fn get_chain(num_shards: NumShards) -> Chain { - get_chain_with_epoch_length_and_num_shards(10, num_shards) - } +use std::sync::Arc; + +use crate::chain::Chain; +use crate::test_utils::KeyValueRuntime; +use crate::types::{ChainGenesis, Tip}; +use crate::DoomslugThresholdMode; +use near_crypto::KeyType; +use near_primitives::block::Block; +use near_primitives::merkle::PartialMerkleTree; +use near_primitives::shard_layout::ShardUId; +use near_primitives::types::{NumBlocks, NumShards, StateRoot}; +use near_primitives::validator_signer::InMemoryValidatorSigner; +use near_store::test_utils::{create_test_store, gen_changes}; +use near_store::{ShardTries, StoreUpdate, Trie, WrappedTrieChanges}; +use rand::Rng; + +fn get_chain(num_shards: NumShards) -> Chain { + get_chain_with_epoch_length_and_num_shards(10, num_shards) +} - fn get_chain_with_epoch_length_and_num_shards( - epoch_length: NumBlocks, - num_shards: NumShards, - ) -> Chain { - let store = create_test_store(); - let chain_genesis = ChainGenesis::test(); - let validators = vec![vec!["test1"]]; - let runtime_adapter = Arc::new(KeyValueRuntime::new_with_validators( - store.clone(), - validators - .into_iter() - .map(|inner| { - inner.into_iter().map(|account_id| account_id.parse().unwrap()).collect() - }) - .collect(), - 1, - num_shards, - epoch_length, - )); - Chain::new(runtime_adapter, &chain_genesis, DoomslugThresholdMode::NoApprovals).unwrap() - } +fn get_chain_with_epoch_length_and_num_shards( + epoch_length: NumBlocks, + num_shards: NumShards, +) -> Chain { + let store = create_test_store(); + let chain_genesis = ChainGenesis::test(); + let validators = vec![vec!["test1"]]; + let runtime_adapter = Arc::new(KeyValueRuntime::new_with_validators( + store.clone(), + validators + .into_iter() + .map(|inner| inner.into_iter().map(|account_id| account_id.parse().unwrap()).collect()) + .collect(), + 1, + num_shards, + epoch_length, + )); + Chain::new(runtime_adapter, &chain_genesis, DoomslugThresholdMode::NoApprovals).unwrap() +} - // Build a chain of num_blocks on top of prev_block - fn do_fork( - mut prev_block: Block, - mut prev_state_roots: Vec, - tries: ShardTries, - chain: &mut Chain, - num_blocks: u64, - states: &mut Vec<(Block, Vec, Vec, Option>)>>)>, - max_changes: usize, - verbose: bool, - ) { - let mut rng = rand::thread_rng(); - let signer = Arc::new(InMemoryValidatorSigner::from_seed( - "test1".parse().unwrap(), - KeyType::ED25519, - "test1", - )); - let num_shards = prev_state_roots.len() as u64; - let runtime_adapter = chain.runtime_adapter.clone(); - for i in 0..num_blocks { - runtime_adapter - .get_next_epoch_id_from_prev_block(prev_block.hash()) - .expect("block must exist"); - let block = Block::empty(&prev_block, &*signer); - - let head = chain.head().unwrap(); - let mut store_update = chain.mut_store().store_update(); - if i == 0 { - store_update - .save_block_merkle_tree(*prev_block.hash(), PartialMerkleTree::default()); - } - store_update.save_block(block.clone()); - store_update.inc_block_refcount(block.header().prev_hash()).unwrap(); - store_update.save_block_header(block.header().clone()).unwrap(); - let tip = Tip::from_header(block.header()); - if head.height < tip.height { - store_update.save_head(&tip).unwrap(); - } +// Build a chain of num_blocks on top of prev_block +fn do_fork( + mut prev_block: Block, + mut prev_state_roots: Vec, + tries: ShardTries, + chain: &mut Chain, + num_blocks: u64, + states: &mut Vec<(Block, Vec, Vec, Option>)>>)>, + max_changes: usize, + verbose: bool, +) { + let mut rng = rand::thread_rng(); + let signer = Arc::new(InMemoryValidatorSigner::from_seed( + "test1".parse().unwrap(), + KeyType::ED25519, + "test1", + )); + let num_shards = prev_state_roots.len() as u64; + let runtime_adapter = chain.runtime_adapter.clone(); + for i in 0..num_blocks { + runtime_adapter + .get_next_epoch_id_from_prev_block(prev_block.hash()) + .expect("block must exist"); + let block = Block::empty(&prev_block, &*signer); + + let head = chain.head().unwrap(); + let mut store_update = chain.mut_store().store_update(); + if i == 0 { + store_update.save_block_merkle_tree(*prev_block.hash(), PartialMerkleTree::default()); + } + store_update.save_block(block.clone()); + store_update.inc_block_refcount(block.header().prev_hash()).unwrap(); + store_update.save_block_header(block.header().clone()).unwrap(); + let tip = Tip::from_header(block.header()); + if head.height < tip.height { + store_update.save_head(&tip).unwrap(); + } - let mut trie_changes_shards = Vec::new(); - for shard_id in 0..num_shards { - let shard_uid = ShardUId { version: 0, shard_id: shard_id as u32 }; - let trie_changes_data = gen_changes(&mut rng, max_changes); - let state_root = prev_state_roots[shard_id as usize]; - let trie = tries.get_trie_for_shard(shard_uid); - let trie_changes = - trie.update(&state_root, trie_changes_data.iter().cloned()).unwrap(); - if verbose { - println!("state new {:?} {:?}", block.header().height(), trie_changes_data); - } - - let new_root = trie_changes.new_root; - let wrapped_trie_changes = WrappedTrieChanges::new( - tries.clone(), - shard_uid, - trie_changes, - Default::default(), - *block.hash(), - ); - store_update.save_trie_changes(wrapped_trie_changes); - - prev_state_roots[shard_id as usize] = new_root; - trie_changes_shards.push(trie_changes_data); + let mut trie_changes_shards = Vec::new(); + for shard_id in 0..num_shards { + let shard_uid = ShardUId { version: 0, shard_id: shard_id as u32 }; + let trie_changes_data = gen_changes(&mut rng, max_changes); + let state_root = prev_state_roots[shard_id as usize]; + let trie = tries.get_trie_for_shard(shard_uid); + let trie_changes = trie.update(&state_root, trie_changes_data.iter().cloned()).unwrap(); + if verbose { + println!("state new {:?} {:?}", block.header().height(), trie_changes_data); } - store_update.commit().unwrap(); - states.push((block.clone(), prev_state_roots.clone(), trie_changes_shards)); - prev_block = block.clone(); + + let new_root = trie_changes.new_root; + let wrapped_trie_changes = WrappedTrieChanges::new( + tries.clone(), + shard_uid, + trie_changes, + Default::default(), + *block.hash(), + ); + store_update.save_trie_changes(wrapped_trie_changes); + + prev_state_roots[shard_id as usize] = new_root; + trie_changes_shards.push(trie_changes_data); } + store_update.commit().unwrap(); + states.push((block.clone(), prev_state_roots.clone(), trie_changes_shards)); + prev_block = block.clone(); } +} - // This test infrastructure do the following: - // Build Chain 1 from the full data, then GC it. - // Build Chain 2 from the data removing everything that should be removed after GC. - // Make sure that Chain 1 == Chain 2. - fn gc_fork_common(simple_chains: Vec, max_changes: usize) { - // Running the test - println!( - "Running gc test: max_changes per block = {:?}, simple_chains_len = {:?} simple_chains = {:?}", +// This test infrastructure do the following: +// Build Chain 1 from the full data, then GC it. +// Build Chain 2 from the data removing everything that should be removed after GC. +// Make sure that Chain 1 == Chain 2. +fn gc_fork_common(simple_chains: Vec, max_changes: usize) { + // Running the test + println!( + "Running gc test: max_changes per block = {:?}, simple_chains_len = {:?} simple_chains = {:?}", + max_changes, + simple_chains.len(), + simple_chains + ); + let verbose = false; + + let num_shards = rand::thread_rng().gen_range(1, 3); + + // Init Chain 1 + let mut chain1 = get_chain(num_shards); + let tries1 = chain1.runtime_adapter.get_tries(); + let mut rng = rand::thread_rng(); + let shard_to_check_trie = rng.gen_range(0, num_shards); + let shard_uid = ShardUId { version: 0, shard_id: shard_to_check_trie as u32 }; + let trie1 = tries1.get_trie_for_shard(shard_uid); + let genesis1 = chain1.get_block_by_height(0).unwrap().clone(); + let mut states1 = vec![]; + states1.push(( + genesis1.clone(), + vec![Trie::empty_root(); num_shards as usize], + vec![Vec::new(); num_shards as usize], + )); + + for simple_chain in simple_chains.iter() { + let (source_block1, state_root1, _) = states1[simple_chain.from as usize].clone(); + do_fork( + source_block1.clone(), + state_root1, + tries1.clone(), + &mut chain1, + simple_chain.length, + &mut states1, max_changes, - simple_chains.len(), - simple_chains + verbose, ); - let verbose = false; - - let num_shards = rand::thread_rng().gen_range(1, 3); - - // Init Chain 1 - let mut chain1 = get_chain(num_shards); - let tries1 = chain1.runtime_adapter.get_tries(); - let mut rng = rand::thread_rng(); - let shard_to_check_trie = rng.gen_range(0, num_shards); - let shard_uid = ShardUId { version: 0, shard_id: shard_to_check_trie as u32 }; - let trie1 = tries1.get_trie_for_shard(shard_uid); - let genesis1 = chain1.get_block_by_height(0).unwrap().clone(); - let mut states1 = vec![]; - states1.push(( - genesis1.clone(), - vec![Trie::empty_root(); num_shards as usize], - vec![Vec::new(); num_shards as usize], - )); - - for simple_chain in simple_chains.iter() { - let (source_block1, state_root1, _) = states1[simple_chain.from as usize].clone(); - do_fork( - source_block1.clone(), - state_root1, - tries1.clone(), - &mut chain1, - simple_chain.length, - &mut states1, - max_changes, - verbose, - ); - } + } - // GC execution - let clear_data = chain1.clear_data(tries1.clone(), 100); - if clear_data.is_err() { - println!("clear data failed = {:?}", clear_data); - assert!(false); - } + // GC execution + let clear_data = chain1.clear_data(tries1.clone(), 100); + if clear_data.is_err() { + println!("clear data failed = {:?}", clear_data); + assert!(false); + } - let mut chain2 = get_chain(num_shards); - let tries2 = chain2.runtime_adapter.get_tries(); - let trie2 = tries2.get_trie_for_shard(shard_uid); + let mut chain2 = get_chain(num_shards); + let tries2 = chain2.runtime_adapter.get_tries(); + let trie2 = tries2.get_trie_for_shard(shard_uid); - // Find gc_height - let mut gc_height = simple_chains[0].length - 51; - for (i, simple_chain) in simple_chains.iter().enumerate() { - if (!simple_chain.is_removed) && gc_height > simple_chain.from && i > 0 { - gc_height = simple_chain.from - } - } - if verbose { - println!("GC height = {:?}", gc_height); + // Find gc_height + let mut gc_height = simple_chains[0].length - 51; + for (i, simple_chain) in simple_chains.iter().enumerate() { + if (!simple_chain.is_removed) && gc_height > simple_chain.from && i > 0 { + gc_height = simple_chain.from } + } + if verbose { + println!("GC height = {:?}", gc_height); + } - let mut start_index = 1; // zero is for genesis - let mut state_roots2 = vec![]; - state_roots2.push(Trie::empty_root()); - - for simple_chain in simple_chains.iter() { - if simple_chain.is_removed { - for _ in 0..simple_chain.length { - // This chain is deleted in Chain1 - state_roots2.push(Trie::empty_root()); - } - start_index += simple_chain.length; - continue; - } + let mut start_index = 1; // zero is for genesis + let mut state_roots2 = vec![]; + state_roots2.push(Trie::empty_root()); - let mut state_root2 = state_roots2[simple_chain.from as usize]; - let state_root1 = states1[simple_chain.from as usize].1[shard_to_check_trie as usize]; - assert!(trie1.iter(&state_root1).is_ok()); - assert_eq!(state_root1, state_root2); - - for i in start_index..start_index + simple_chain.length { - let mut store_update2 = chain2.mut_store().store_update(); - let (block1, state_root1, changes1) = states1[i as usize].clone(); - // Apply to Trie 2 the same changes (changes1) as applied to Trie 1 - let trie_changes2 = trie2 - .update(&state_root2, changes1[shard_to_check_trie as usize].iter().cloned()) - .unwrap(); - // i == gc_height is the only height should be processed here - if block1.header().height() > gc_height || i == gc_height { - let mut trie_store_update2 = StoreUpdate::new_with_tries(tries2.clone()); - tries2 - .apply_insertions(&trie_changes2, shard_uid, &mut trie_store_update2) - .unwrap(); - state_root2 = trie_changes2.new_root; - assert_eq!(state_root1[shard_to_check_trie as usize], state_root2); - store_update2.merge(trie_store_update2); - } else { - let (trie_store_update2, new_root2) = - tries2.apply_all(&trie_changes2, shard_uid).unwrap(); - state_root2 = new_root2; - store_update2.merge(trie_store_update2); - } - state_roots2.push(state_root2); - store_update2.commit().unwrap(); + for simple_chain in simple_chains.iter() { + if simple_chain.is_removed { + for _ in 0..simple_chain.length { + // This chain is deleted in Chain1 + state_roots2.push(Trie::empty_root()); } start_index += simple_chain.length; + continue; } - let mut start_index = 1; // zero is for genesis - for simple_chain in simple_chains.iter() { - if simple_chain.is_removed { - start_index += simple_chain.length; - continue; - } - for i in start_index..start_index + simple_chain.length { - let (block1, state_root1, _) = states1[i as usize].clone(); - let state_root1 = state_root1[shard_to_check_trie as usize]; - if block1.header().height() > gc_height || i == gc_height { - assert!(trie1.iter(&state_root1).is_ok()); - assert!(trie2.iter(&state_root1).is_ok()); - let a = trie1 - .iter(&state_root1) - .unwrap() - .map(|item| item.unwrap().0) - .collect::>(); - let b = trie2 - .iter(&state_root1) - .unwrap() - .map(|item| item.unwrap().0) - .collect::>(); - assert_eq!(a, b); - } + let mut state_root2 = state_roots2[simple_chain.from as usize]; + let state_root1 = states1[simple_chain.from as usize].1[shard_to_check_trie as usize]; + assert!(trie1.iter(&state_root1).is_ok()); + assert_eq!(state_root1, state_root2); + + for i in start_index..start_index + simple_chain.length { + let mut store_update2 = chain2.mut_store().store_update(); + let (block1, state_root1, changes1) = states1[i as usize].clone(); + // Apply to Trie 2 the same changes (changes1) as applied to Trie 1 + let trie_changes2 = trie2 + .update(&state_root2, changes1[shard_to_check_trie as usize].iter().cloned()) + .unwrap(); + // i == gc_height is the only height should be processed here + if block1.header().height() > gc_height || i == gc_height { + let mut trie_store_update2 = StoreUpdate::new_with_tries(tries2.clone()); + tries2 + .apply_insertions(&trie_changes2, shard_uid, &mut trie_store_update2) + .unwrap(); + state_root2 = trie_changes2.new_root; + assert_eq!(state_root1[shard_to_check_trie as usize], state_root2); + store_update2.merge(trie_store_update2); + } else { + let (trie_store_update2, new_root2) = + tries2.apply_all(&trie_changes2, shard_uid).unwrap(); + state_root2 = new_root2; + store_update2.merge(trie_store_update2); } - start_index += simple_chain.length; + state_roots2.push(state_root2); + store_update2.commit().unwrap(); } + start_index += simple_chain.length; } - // from is an index in blocks array, length is the number of blocks in a chain on top of blocks[from], - // is_removed = should this chain be removed after GC - #[derive(Debug, Clone)] - pub struct SimpleChain { - from: u64, - length: u64, - is_removed: bool, - } - - // This test creates only chain - #[test] - fn test_gc_remove_simple_chain_sanity() { - for max_changes in 1..=20 { - let chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - // remove 50 blocks, height = 50 will be the earliest one which exists - gc_fork_common(chains, max_changes); + let mut start_index = 1; // zero is for genesis + for simple_chain in simple_chains.iter() { + if simple_chain.is_removed { + start_index += simple_chain.length; + continue; } - } - - // Creates simple shorter fork and GCs it - fn test_gc_remove_fork_common(max_changes_limit: usize) { - for max_changes in 1..=max_changes_limit { - for fork_length in 1..=10 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: fork_length, is_removed: true }, - ]; - gc_fork_common(chains, max_changes); - } - for fork_length in 30..=40 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: fork_length, is_removed: true }, - ]; - gc_fork_common(chains, max_changes); + for i in start_index..start_index + simple_chain.length { + let (block1, state_root1, _) = states1[i as usize].clone(); + let state_root1 = state_root1[shard_to_check_trie as usize]; + if block1.header().height() > gc_height || i == gc_height { + assert!(trie1.iter(&state_root1).is_ok()); + assert!(trie2.iter(&state_root1).is_ok()); + let a = trie1 + .iter(&state_root1) + .unwrap() + .map(|item| item.unwrap().0) + .collect::>(); + let b = trie2 + .iter(&state_root1) + .unwrap() + .map(|item| item.unwrap().0) + .collect::>(); + assert_eq!(a, b); } } + start_index += simple_chain.length; } +} - #[test] - fn test_gc_remove_fork_small() { - test_gc_remove_fork_common(1) - } +// from is an index in blocks array, length is the number of blocks in a chain on top of blocks[from], +// is_removed = should this chain be removed after GC +#[derive(Debug, Clone)] +pub struct SimpleChain { + from: u64, + length: u64, + is_removed: bool, +} - #[cfg(feature = "expensive_tests")] - #[test] - fn test_gc_remove_fork_large() { - test_gc_remove_fork_common(20) +// This test creates only chain +#[test] +fn test_gc_remove_simple_chain_sanity() { + for max_changes in 1..=20 { + let chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; + // remove 50 blocks, height = 50 will be the earliest one which exists + gc_fork_common(chains, max_changes); } +} - #[test] - fn test_gc_remove_fork_fail_often() { - for _tries in 0..10 { +// Creates simple shorter fork and GCs it +fn test_gc_remove_fork_common(max_changes_limit: usize) { + for max_changes in 1..=max_changes_limit { + for fork_length in 1..=10 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: 35, is_removed: true }, + SimpleChain { from: 10, length: fork_length, is_removed: true }, ]; - gc_fork_common(chains, 1); + gc_fork_common(chains, max_changes); + } + for fork_length in 30..=40 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: 80, is_removed: false }, + SimpleChain { from: 10, length: fork_length, is_removed: true }, ]; - gc_fork_common(chains, 6); + gc_fork_common(chains, max_changes); } } +} - // Creates simple shorter fork and NOT GCs it - fn test_gc_not_remove_fork_common(max_changes_limit: usize) { - for max_changes in 1..=max_changes_limit { - for fork_length in 41..=50 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: fork_length, is_removed: false }, - ]; - gc_fork_common(chains, max_changes); - } - for fork_length in 80..=90 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: fork_length, is_removed: false }, - ]; - gc_fork_common(chains, max_changes); - } - } - } +#[test] +fn test_gc_remove_fork_small() { + test_gc_remove_fork_common(1) +} - #[test] - fn test_gc_not_remove_fork_small() { - test_gc_not_remove_fork_common(1) - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_gc_remove_fork_large() { + test_gc_remove_fork_common(20) +} - #[cfg(feature = "expensive_tests")] - #[test] - fn test_gc_not_remove_fork_large() { - test_gc_not_remove_fork_common(20) +#[test] +fn test_gc_remove_fork_fail_often() { + for _tries in 0..10 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 10, length: 35, is_removed: true }, + ]; + gc_fork_common(chains, 1); + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 10, length: 80, is_removed: false }, + ]; + gc_fork_common(chains, 6); } +} - // Creates simple longer fork and NOT GCs it - #[test] - fn test_gc_not_remove_longer_fork() { - for fork_length in 91..=100 { +// Creates simple shorter fork and NOT GCs it +fn test_gc_not_remove_fork_common(max_changes_limit: usize) { + for max_changes in 1..=max_changes_limit { + for fork_length in 41..=50 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, SimpleChain { from: 10, length: fork_length, is_removed: false }, ]; - gc_fork_common(chains, 1); - } - } - - // This test creates forks from genesis - #[test] - fn test_gc_forks_from_genesis() { - for fork_length in 1..=10 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 0, length: fork_length, is_removed: true }, - ]; - gc_fork_common(chains, 1); - } - for fork_length in 45..=50 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 0, length: fork_length, is_removed: true }, - ]; - gc_fork_common(chains, 1); - } - for fork_length in 51..=55 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 0, length: fork_length, is_removed: false }, - ]; - gc_fork_common(chains, 1); + gc_fork_common(chains, max_changes); } - for fork_length in 0..=10 { + for fork_length in 80..=90 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 0, length: 51 + fork_length, is_removed: false }, - SimpleChain { from: 0, length: fork_length, is_removed: true }, - SimpleChain { from: 0, length: 50 - fork_length, is_removed: true }, + SimpleChain { from: 10, length: fork_length, is_removed: false }, ]; - gc_fork_common(chains, 1); + gc_fork_common(chains, max_changes); } } +} - #[test] - fn test_gc_overlap() { - for max_changes in 1..=20 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 10, length: 70, is_removed: false }, - SimpleChain { from: 20, length: 25, is_removed: true }, - SimpleChain { from: 30, length: 30, is_removed: false }, - SimpleChain { from: 40, length: 1, is_removed: true }, - ]; - gc_fork_common(chains, max_changes); - } +#[test] +fn test_gc_not_remove_fork_small() { + test_gc_not_remove_fork_common(1) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_gc_not_remove_fork_large() { + test_gc_not_remove_fork_common(20) +} + +// Creates simple longer fork and NOT GCs it +#[test] +fn test_gc_not_remove_longer_fork() { + for fork_length in 91..=100 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 10, length: fork_length, is_removed: false }, + ]; + gc_fork_common(chains, 1); + } +} + +// This test creates forks from genesis +#[test] +fn test_gc_forks_from_genesis() { + for fork_length in 1..=10 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 0, length: fork_length, is_removed: true }, + ]; + gc_fork_common(chains, 1); } + for fork_length in 45..=50 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 0, length: fork_length, is_removed: true }, + ]; + gc_fork_common(chains, 1); + } + for fork_length in 51..=55 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 0, length: fork_length, is_removed: false }, + ]; + gc_fork_common(chains, 1); + } + for fork_length in 0..=10 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 0, length: 51 + fork_length, is_removed: false }, + SimpleChain { from: 0, length: fork_length, is_removed: true }, + SimpleChain { from: 0, length: 50 - fork_length, is_removed: true }, + ]; + gc_fork_common(chains, 1); + } +} + +#[test] +fn test_gc_overlap() { + for max_changes in 1..=20 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: 10, length: 70, is_removed: false }, + SimpleChain { from: 20, length: 25, is_removed: true }, + SimpleChain { from: 30, length: 30, is_removed: false }, + SimpleChain { from: 40, length: 1, is_removed: true }, + ]; + gc_fork_common(chains, max_changes); + } +} - fn test_gc_boundaries_common(max_changes_limit: usize) { - for max_changes in 1..=max_changes_limit { - for i in 45..=51 { - for len in 1..=5 { - let chains = vec![ - SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: i, length: len, is_removed: i + len <= 50 }, - ]; - gc_fork_common(chains, max_changes); - } +fn test_gc_boundaries_common(max_changes_limit: usize) { + for max_changes in 1..=max_changes_limit { + for i in 45..=51 { + for len in 1..=5 { + let chains = vec![ + SimpleChain { from: 0, length: 101, is_removed: false }, + SimpleChain { from: i, length: len, is_removed: i + len <= 50 }, + ]; + gc_fork_common(chains, max_changes); } } } +} - #[test] - fn test_gc_boundaries_small() { - test_gc_boundaries_common(1) +#[test] +fn test_gc_boundaries_small() { + test_gc_boundaries_common(1) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_gc_boundaries_large() { + test_gc_boundaries_common(20) +} + +fn test_gc_random_common(runs: u64) { + let mut rng = rand::thread_rng(); + for _tests in 0..runs { + let canonical_len = 100 + rng.gen_range(0, 20); + let mut chains = vec![SimpleChain { from: 0, length: canonical_len, is_removed: false }]; + for _num_chains in 1..10 { + let from = rng.gen_range(0, 50); + let len = rng.gen_range(0, 50) + 1; + chains.push(SimpleChain { + from, + length: len, + is_removed: from + len < canonical_len - 50, + }); + gc_fork_common(chains.clone(), rng.gen_range(0, 20) + 1); + } } +} + +#[test] +fn test_gc_random_small() { + test_gc_random_common(3); +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_gc_random_large() { + test_gc_random_common(25); +} - #[cfg(feature = "expensive_tests")] - #[test] - fn test_gc_boundaries_large() { - test_gc_boundaries_common(20) +#[test] +fn test_gc_pine_small() { + let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; + for i in 1..50 { + chains.push(SimpleChain { from: i, length: 1, is_removed: true }); } + for i in 50..100 { + chains.push(SimpleChain { from: i, length: 1, is_removed: false }); + } + gc_fork_common(chains, 3); - fn test_gc_random_common(runs: u64) { - let mut rng = rand::thread_rng(); - for _tests in 0..runs { - let canonical_len = 100 + rng.gen_range(0, 20); - let mut chains = - vec![SimpleChain { from: 0, length: canonical_len, is_removed: false }]; - for _num_chains in 1..10 { - let from = rng.gen_range(0, 50); - let len = rng.gen_range(0, 50) + 1; - chains.push(SimpleChain { - from, - length: len, - is_removed: from + len < canonical_len - 50, - }); - gc_fork_common(chains.clone(), rng.gen_range(0, 20) + 1); - } - } + let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; + for i in 1..49 { + chains.push(SimpleChain { from: i, length: 2, is_removed: true }); } + for i in 49..99 { + chains.push(SimpleChain { from: i, length: 2, is_removed: false }); + } + gc_fork_common(chains, 2); - #[test] - fn test_gc_random_small() { - test_gc_random_common(3); + let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; + for i in 1..48 { + chains.push(SimpleChain { from: i, length: 3, is_removed: true }); + } + for i in 48..98 { + chains.push(SimpleChain { from: i, length: 3, is_removed: false }); } + gc_fork_common(chains, 1); - #[cfg(feature = "expensive_tests")] - #[test] - fn test_gc_random_large() { - test_gc_random_common(25); + let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; + for i in 1..40 { + chains.push(SimpleChain { from: i, length: 11, is_removed: true }); } + for i in 40..90 { + chains.push(SimpleChain { from: i, length: 11, is_removed: false }); + } + gc_fork_common(chains, 1); +} - #[test] - fn test_gc_pine_small() { +#[cfg(feature = "expensive_tests")] +#[test] +fn test_gc_pine() { + for max_changes in 1..=20 { let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; for i in 1..50 { chains.push(SimpleChain { from: i, length: 1, is_removed: true }); @@ -477,25 +511,7 @@ mod tests { for i in 50..100 { chains.push(SimpleChain { from: i, length: 1, is_removed: false }); } - gc_fork_common(chains, 3); - - let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..49 { - chains.push(SimpleChain { from: i, length: 2, is_removed: true }); - } - for i in 49..99 { - chains.push(SimpleChain { from: i, length: 2, is_removed: false }); - } - gc_fork_common(chains, 2); - - let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..48 { - chains.push(SimpleChain { from: i, length: 3, is_removed: true }); - } - for i in 48..98 { - chains.push(SimpleChain { from: i, length: 3, is_removed: false }); - } - gc_fork_common(chains, 1); + gc_fork_common(chains, max_changes); let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; for i in 1..40 { @@ -504,54 +520,30 @@ mod tests { for i in 40..90 { chains.push(SimpleChain { from: i, length: 11, is_removed: false }); } - gc_fork_common(chains, 1); + gc_fork_common(chains, max_changes); } +} - #[cfg(feature = "expensive_tests")] - #[test] - fn test_gc_pine() { - for max_changes in 1..=20 { - let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..50 { - chains.push(SimpleChain { from: i, length: 1, is_removed: true }); - } - for i in 50..100 { - chains.push(SimpleChain { from: i, length: 1, is_removed: false }); - } - gc_fork_common(chains, max_changes); - - let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..40 { - chains.push(SimpleChain { from: i, length: 11, is_removed: true }); - } - for i in 40..90 { - chains.push(SimpleChain { from: i, length: 11, is_removed: false }); - } - gc_fork_common(chains, max_changes); +fn test_gc_star_common(max_changes_limit: usize) { + for max_changes in 1..=max_changes_limit { + let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; + for i in 1..=17 { + chains.push(SimpleChain { from: 33, length: i, is_removed: true }); } - } - - fn test_gc_star_common(max_changes_limit: usize) { - for max_changes in 1..=max_changes_limit { - let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..=17 { - chains.push(SimpleChain { from: 33, length: i, is_removed: true }); - } - for i in 18..67 { - chains.push(SimpleChain { from: 33, length: i, is_removed: false }); - } - gc_fork_common(chains, max_changes); + for i in 18..67 { + chains.push(SimpleChain { from: 33, length: i, is_removed: false }); } + gc_fork_common(chains, max_changes); } +} - #[test] - fn test_gc_star_small() { - test_gc_star_common(1) - } +#[test] +fn test_gc_star_small() { + test_gc_star_common(1) +} - #[cfg(feature = "expensive_tests")] - #[test] - fn test_gc_star_large() { - test_gc_star_common(20) - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_gc_star_large() { + test_gc_star_common(20) } diff --git a/chain/chain/src/tests/mod.rs b/chain/chain/src/tests/mod.rs index 08618941894..490626effbf 100644 --- a/chain/chain/src/tests/mod.rs +++ b/chain/chain/src/tests/mod.rs @@ -1,4 +1,5 @@ mod challenges; +#[cfg(feature = "expensive_tests")] mod doomslug; mod gc; mod simple_chain; diff --git a/chain/client/src/tests/catching_up.rs b/chain/client/src/tests/catching_up.rs index 476fab5a4b6..9cc2457a7a7 100644 --- a/chain/client/src/tests/catching_up.rs +++ b/chain/client/src/tests/catching_up.rs @@ -1,434 +1,433 @@ -#[cfg(test)] -#[cfg(feature = "expensive_tests")] -mod tests { - use std::collections::hash_map::Entry; - use std::collections::{HashMap, HashSet}; - use std::sync::{Arc, RwLock}; - - use actix::{Addr, System}; - use borsh::{BorshDeserialize, BorshSerialize}; - use futures::{future, FutureExt}; - - use crate::test_utils::setup_mock_all_validators; - use crate::{ClientActor, Query, ViewClientActor}; - use near_actix_test_utils::run_actix; - use near_chain::test_utils::account_id_to_shard_id; - use near_chain_configs::TEST_STATE_SYNC_TIMEOUT; - use near_crypto::{InMemorySigner, KeyType}; - use near_logger_utils::init_integration_logger; - use near_network::types::{ - NetworkClientMessages, NetworkRequests, NetworkResponses, PeerManagerMessageRequest, - }; - use near_network_primitives::types::{ - AccountIdOrPeerTrackingShard, AccountOrPeerIdOrHash, PeerInfo, - }; - use near_primitives::hash::{hash as hash_func, CryptoHash}; - use near_primitives::receipt::Receipt; - use near_primitives::sharding::ChunkHash; - use near_primitives::transaction::SignedTransaction; - use near_primitives::types::{AccountId, BlockHeight, BlockHeightDelta, BlockReference}; - use near_primitives::views::QueryRequest; - use near_primitives::views::QueryResponseKind::ViewAccount; - - fn get_validators_and_key_pairs() -> (Vec>, Vec) { - let validators: Vec> = vec![ - ["test1.1", "test1.2", "test1.3", "test1.4"] - .iter() - .map(|account_id| account_id.parse().unwrap()) - .collect(), - ["test2.1", "test2.2", "test2.3", "test2.4"] - .iter() - .map(|account_id| account_id.parse().unwrap()) - .collect(), - [ - "test3.1", "test3.2", "test3.3", "test3.4", "test3.5", "test3.6", "test3.7", - "test3.8", - ] +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, RwLock}; + +use actix::{Addr, System}; +use borsh::{BorshDeserialize, BorshSerialize}; +use futures::{future, FutureExt}; + +use crate::test_utils::setup_mock_all_validators; +use crate::{ClientActor, Query, ViewClientActor}; +use near_actix_test_utils::run_actix; +use near_chain::test_utils::account_id_to_shard_id; +use near_chain_configs::TEST_STATE_SYNC_TIMEOUT; +use near_crypto::{InMemorySigner, KeyType}; +use near_logger_utils::init_integration_logger; +use near_network::types::{ + NetworkClientMessages, NetworkRequests, NetworkResponses, PeerManagerMessageRequest, +}; +use near_network_primitives::types::{ + AccountIdOrPeerTrackingShard, AccountOrPeerIdOrHash, PeerInfo, +}; +use near_primitives::hash::{hash as hash_func, CryptoHash}; +use near_primitives::receipt::Receipt; +use near_primitives::sharding::ChunkHash; +use near_primitives::transaction::SignedTransaction; +use near_primitives::types::{AccountId, BlockHeight, BlockHeightDelta, BlockReference}; +use near_primitives::views::QueryRequest; +use near_primitives::views::QueryResponseKind::ViewAccount; + +fn get_validators_and_key_pairs() -> (Vec>, Vec) { + let validators: Vec> = vec![ + ["test1.1", "test1.2", "test1.3", "test1.4"] .iter() .map(|account_id| account_id.parse().unwrap()) .collect(), - ]; - let key_pairs = vec![ - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), // 4 - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), // 8 - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), - PeerInfo::random(), // 16 - ]; - (validators, key_pairs) - } - - fn send_tx( - connector: &Addr, - from: AccountId, - to: AccountId, - amount: u128, - nonce: u64, - block_hash: CryptoHash, - ) { - let signer = InMemorySigner::from_seed("test1".parse().unwrap(), KeyType::ED25519, "test1"); - connector.do_send(NetworkClientMessages::Transaction { - transaction: SignedTransaction::send_money( - nonce, from, to, &signer, amount, block_hash, - ), - is_forwarded: false, - check_only: false, - }); - } - - enum ReceiptsSyncPhases { - WaitingForFirstBlock, - WaitingForSecondBlock, - WaitingForDistantEpoch, - VerifyingOutgoingReceipts, - WaitingForValidate, - } - - #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] - pub struct StateRequestStruct { - pub shard_id: u64, - pub sync_hash: CryptoHash, - pub part_id: Option, - pub target: AccountOrPeerIdOrHash, - } - - /// Sanity checks that the incoming and outgoing receipts are properly sent and received - #[test] - fn test_catchup_receipts_sync_third_epoch() { - test_catchup_receipts_sync_common(13, 1, false) - } - - /// The test aggressively blocks lots of state requests - /// and causes at least two timeouts per node (first for header, second for parts). - /// - /// WARNING! For your convenience, set manually STATE_SYNC_TIMEOUT to 1 before running the test. - /// It will be executed 10 times faster. - /// The reason of increasing block_prod_time in the test is to allow syncing complete. - /// Otherwise epochs will be changing faster than state sync happen. - #[test] - fn test_catchup_receipts_sync_hold() { - test_catchup_receipts_sync_common(13, 1, true) - } - - #[test] - fn test_catchup_receipts_sync_last_block() { - test_catchup_receipts_sync_common(13, 5, false) - } - - #[test] - fn test_catchup_receipts_sync_distant_epoch() { - test_catchup_receipts_sync_common(35, 1, false) - } - - fn test_catchup_receipts_sync_common(wait_till: u64, send: u64, sync_hold: bool) { - let validator_groups = 1; - init_integration_logger(); - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - - let (validators, key_pairs) = get_validators_and_key_pairs(); - - let phase = Arc::new(RwLock::new(ReceiptsSyncPhases::WaitingForFirstBlock)); - let seen_heights_with_receipts = Arc::new(RwLock::new(HashSet::::new())); - let seen_hashes_with_state = Arc::new(RwLock::new(HashSet::::new())); - - let connectors1 = connectors.clone(); - let mut block_prod_time: u64 = 3200; - if sync_hold { - block_prod_time *= TEST_STATE_SYNC_TIMEOUT as u64; - } - let (_, conn, _) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - validator_groups, - true, - block_prod_time, - false, - false, - 5, - false, - vec![true; validators.iter().map(|x| x.len()).sum()], - vec![false; validators.iter().map(|x| x.len()).sum()], - false, - Arc::new(RwLock::new(Box::new( - move |_account_id: _, msg: &PeerManagerMessageRequest| { - let msg = msg.as_network_requests_ref(); - let account_from = "test3.3".parse().unwrap(); - let account_to = "test1.1".parse().unwrap(); - let source_shard_id = account_id_to_shard_id(&account_from, 4); - let destination_shard_id = account_id_to_shard_id(&account_to, 4); - - let mut phase = phase.write().unwrap(); - let mut seen_heights_with_receipts = - seen_heights_with_receipts.write().unwrap(); - let mut seen_hashes_with_state = seen_hashes_with_state.write().unwrap(); - match *phase { - ReceiptsSyncPhases::WaitingForFirstBlock => { - if let NetworkRequests::Block { block } = msg { - assert!(block.header().height() <= send); - // This tx is rather fragile, specifically it's important that - // 1. the `from` and `to` account are not in the same shard; - // 2. ideally the producer of the chunk at height 3 for the shard - // in which `from` resides should not also be a block producer - // at height 3 - // 3. The `from` shard should also not match the block producer - // for height 1, because such block producer will produce - // the chunk for height 2 right away, before we manage to send - // the transaction. - if block.header().height() == send { - println!( - "From shard: {}, to shard: {}", - source_shard_id, destination_shard_id, + ["test2.1", "test2.2", "test2.3", "test2.4"] + .iter() + .map(|account_id| account_id.parse().unwrap()) + .collect(), + ["test3.1", "test3.2", "test3.3", "test3.4", "test3.5", "test3.6", "test3.7", "test3.8"] + .iter() + .map(|account_id| account_id.parse().unwrap()) + .collect(), + ]; + let key_pairs = vec![ + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), // 4 + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), // 8 + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), + PeerInfo::random(), // 16 + ]; + (validators, key_pairs) +} + +fn send_tx( + connector: &Addr, + from: AccountId, + to: AccountId, + amount: u128, + nonce: u64, + block_hash: CryptoHash, +) { + let signer = InMemorySigner::from_seed("test1".parse().unwrap(), KeyType::ED25519, "test1"); + connector.do_send(NetworkClientMessages::Transaction { + transaction: SignedTransaction::send_money(nonce, from, to, &signer, amount, block_hash), + is_forwarded: false, + check_only: false, + }); +} + +enum ReceiptsSyncPhases { + WaitingForFirstBlock, + WaitingForSecondBlock, + WaitingForDistantEpoch, + VerifyingOutgoingReceipts, + WaitingForValidate, +} + +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub struct StateRequestStruct { + pub shard_id: u64, + pub sync_hash: CryptoHash, + pub part_id: Option, + pub target: AccountOrPeerIdOrHash, +} + +/// Sanity checks that the incoming and outgoing receipts are properly sent and received +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_receipts_sync_third_epoch() { + test_catchup_receipts_sync_common(13, 1, false) +} + +/// The test aggressively blocks lots of state requests +/// and causes at least two timeouts per node (first for header, second for parts). +/// +/// WARNING! For your convenience, set manually STATE_SYNC_TIMEOUT to 1 before running the test. +/// It will be executed 10 times faster. +/// The reason of increasing block_prod_time in the test is to allow syncing complete. +/// Otherwise epochs will be changing faster than state sync happen. +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_receipts_sync_hold() { + test_catchup_receipts_sync_common(13, 1, true) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_receipts_sync_last_block() { + test_catchup_receipts_sync_common(13, 5, false) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_receipts_sync_distant_epoch() { + test_catchup_receipts_sync_common(35, 1, false) +} + +fn test_catchup_receipts_sync_common(wait_till: u64, send: u64, sync_hold: bool) { + let validator_groups = 1; + init_integration_logger(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); + + let (validators, key_pairs) = get_validators_and_key_pairs(); + + let phase = Arc::new(RwLock::new(ReceiptsSyncPhases::WaitingForFirstBlock)); + let seen_heights_with_receipts = Arc::new(RwLock::new(HashSet::::new())); + let seen_hashes_with_state = Arc::new(RwLock::new(HashSet::::new())); + + let connectors1 = connectors.clone(); + let mut block_prod_time: u64 = 3200; + if sync_hold { + block_prod_time *= TEST_STATE_SYNC_TIMEOUT as u64; + } + let (_, conn, _) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + validator_groups, + true, + block_prod_time, + false, + false, + 5, + false, + vec![true; validators.iter().map(|x| x.len()).sum()], + vec![false; validators.iter().map(|x| x.len()).sum()], + false, + Arc::new(RwLock::new(Box::new( + move |_account_id: _, msg: &PeerManagerMessageRequest| { + let msg = msg.as_network_requests_ref(); + let account_from = "test3.3".parse().unwrap(); + let account_to = "test1.1".parse().unwrap(); + let source_shard_id = account_id_to_shard_id(&account_from, 4); + let destination_shard_id = account_id_to_shard_id(&account_to, 4); + + let mut phase = phase.write().unwrap(); + let mut seen_heights_with_receipts = + seen_heights_with_receipts.write().unwrap(); + let mut seen_hashes_with_state = seen_hashes_with_state.write().unwrap(); + match *phase { + ReceiptsSyncPhases::WaitingForFirstBlock => { + if let NetworkRequests::Block { block } = msg { + assert!(block.header().height() <= send); + // This tx is rather fragile, specifically it's important that + // 1. the `from` and `to` account are not in the same shard; + // 2. ideally the producer of the chunk at height 3 for the shard + // in which `from` resides should not also be a block producer + // at height 3 + // 3. The `from` shard should also not match the block producer + // for height 1, because such block producer will produce + // the chunk for height 2 right away, before we manage to send + // the transaction. + if block.header().height() == send { + println!( + "From shard: {}, to shard: {}", + source_shard_id, destination_shard_id, + ); + for i in 0..16 { + send_tx( + &connectors1.write().unwrap()[i].0, + account_from.clone(), + account_to.clone(), + 111, + 1, + *block.header().prev_hash(), ); - for i in 0..16 { - send_tx( - &connectors1.write().unwrap()[i].0, - account_from.clone(), - account_to.clone(), - 111, - 1, - *block.header().prev_hash(), - ); - } - *phase = ReceiptsSyncPhases::WaitingForSecondBlock; } + *phase = ReceiptsSyncPhases::WaitingForSecondBlock; } } - ReceiptsSyncPhases::WaitingForSecondBlock => { - // This block now contains a chunk with the transaction sent above. - if let NetworkRequests::Block { block } = msg { - assert!(block.header().height() <= send + 1); - if block.header().height() == send + 1 { - *phase = ReceiptsSyncPhases::WaitingForDistantEpoch; - } + } + ReceiptsSyncPhases::WaitingForSecondBlock => { + // This block now contains a chunk with the transaction sent above. + if let NetworkRequests::Block { block } = msg { + assert!(block.header().height() <= send + 1); + if block.header().height() == send + 1 { + *phase = ReceiptsSyncPhases::WaitingForDistantEpoch; } } - ReceiptsSyncPhases::WaitingForDistantEpoch => { - // This block now contains a chunk with the transaction sent above. - if let NetworkRequests::Block { block } = msg { - assert!(block.header().height() >= send + 1); - assert!(block.header().height() <= wait_till); - if block.header().height() == wait_till { - *phase = ReceiptsSyncPhases::VerifyingOutgoingReceipts; - } + } + ReceiptsSyncPhases::WaitingForDistantEpoch => { + // This block now contains a chunk with the transaction sent above. + if let NetworkRequests::Block { block } = msg { + assert!(block.header().height() >= send + 1); + assert!(block.header().height() <= wait_till); + if block.header().height() == wait_till { + *phase = ReceiptsSyncPhases::VerifyingOutgoingReceipts; } - if let NetworkRequests::PartialEncodedChunkMessage { - partial_encoded_chunk, - .. - } = msg - { - // The chunk producers in all epochs before `distant` need to be trying to - // include the receipt. The `distant` epoch is the first one that - // will get the receipt through the state sync. - let receipts: Vec = partial_encoded_chunk - .receipts - .iter() - .map(|x| x.0.clone()) - .flatten() - .collect(); - if receipts.len() > 0 { - assert_eq!( - partial_encoded_chunk.header.shard_id(), - source_shard_id - ); - seen_heights_with_receipts - .insert(partial_encoded_chunk.header.height_created()); - } else { - assert_ne!( - partial_encoded_chunk.header.shard_id(), - source_shard_id - ); - } - // Do not propagate any one parts, this will prevent any chunk from - // being included in the block - return (NetworkResponses::NoResponse.into(), false); + } + if let NetworkRequests::PartialEncodedChunkMessage { + partial_encoded_chunk, + .. + } = msg + { + // The chunk producers in all epochs before `distant` need to be trying to + // include the receipt. The `distant` epoch is the first one that + // will get the receipt through the state sync. + let receipts: Vec = partial_encoded_chunk + .receipts + .iter() + .map(|x| x.0.clone()) + .flatten() + .collect(); + if receipts.len() > 0 { + assert_eq!( + partial_encoded_chunk.header.shard_id(), + source_shard_id + ); + seen_heights_with_receipts + .insert(partial_encoded_chunk.header.height_created()); + } else { + assert_ne!( + partial_encoded_chunk.header.shard_id(), + source_shard_id + ); } - if let NetworkRequests::StateRequestHeader { - shard_id, - sync_hash, - target, - } = msg - { - if sync_hold { - let srs = StateRequestStruct { - shard_id: *shard_id, - sync_hash: *sync_hash, - part_id: None, - target: target.clone(), - }; - if !seen_hashes_with_state - .contains(&hash_func(&srs.try_to_vec().unwrap())) - { - seen_hashes_with_state - .insert(hash_func(&srs.try_to_vec().unwrap())); - return (NetworkResponses::NoResponse.into(), false); - } + // Do not propagate any one parts, this will prevent any chunk from + // being included in the block + return (NetworkResponses::NoResponse.into(), false); + } + if let NetworkRequests::StateRequestHeader { + shard_id, + sync_hash, + target, + } = msg + { + if sync_hold { + let srs = StateRequestStruct { + shard_id: *shard_id, + sync_hash: *sync_hash, + part_id: None, + target: target.clone(), + }; + if !seen_hashes_with_state + .contains(&hash_func(&srs.try_to_vec().unwrap())) + { + seen_hashes_with_state + .insert(hash_func(&srs.try_to_vec().unwrap())); + return (NetworkResponses::NoResponse.into(), false); } } - if let NetworkRequests::StateRequestPart { - shard_id, - sync_hash, - part_id, - target, - } = msg - { - if sync_hold { - let srs = StateRequestStruct { - shard_id: *shard_id, - sync_hash: *sync_hash, - part_id: Some(*part_id), - target: target.clone(), - }; - if !seen_hashes_with_state - .contains(&hash_func(&srs.try_to_vec().unwrap())) - { - seen_hashes_with_state - .insert(hash_func(&srs.try_to_vec().unwrap())); - return (NetworkResponses::NoResponse.into(), false); - } + } + if let NetworkRequests::StateRequestPart { + shard_id, + sync_hash, + part_id, + target, + } = msg + { + if sync_hold { + let srs = StateRequestStruct { + shard_id: *shard_id, + sync_hash: *sync_hash, + part_id: Some(*part_id), + target: target.clone(), + }; + if !seen_hashes_with_state + .contains(&hash_func(&srs.try_to_vec().unwrap())) + { + seen_hashes_with_state + .insert(hash_func(&srs.try_to_vec().unwrap())); + return (NetworkResponses::NoResponse.into(), false); } } } - ReceiptsSyncPhases::VerifyingOutgoingReceipts => { - for height in send + 2..=wait_till { - println!( - "checking height {:?} out of {:?}, result = {:?}", - height, - wait_till, - seen_heights_with_receipts.contains(&height) - ); - if !sync_hold { - // If we don't delay the state, all heights should contain the same receipts - assert!(seen_heights_with_receipts.contains(&height)); - } + } + ReceiptsSyncPhases::VerifyingOutgoingReceipts => { + for height in send + 2..=wait_till { + println!( + "checking height {:?} out of {:?}, result = {:?}", + height, + wait_till, + seen_heights_with_receipts.contains(&height) + ); + if !sync_hold { + // If we don't delay the state, all heights should contain the same receipts + assert!(seen_heights_with_receipts.contains(&height)); } - *phase = ReceiptsSyncPhases::WaitingForValidate; } - ReceiptsSyncPhases::WaitingForValidate => { - // This block now contains a chunk with the transaction sent above. - if let NetworkRequests::Block { block } = msg { - assert!(block.header().height() >= wait_till); - assert!(block.header().height() <= wait_till + 20); - if block.header().height() == wait_till + 20 { - System::current().stop(); - } - if block.header().height() == wait_till + 10 { - for i in 0..16 { - actix::spawn( - connectors1.write().unwrap()[i] - .1 - .send(Query::new( - BlockReference::latest(), - QueryRequest::ViewAccount { - account_id: account_to.clone(), - }, - )) - .then(move |res| { - let res_inner = res.unwrap(); - if let Ok(query_response) = res_inner { - if let ViewAccount( - view_account_result, - ) = query_response.kind - { - assert_eq!( - view_account_result.amount, - 1111 - ); - } + *phase = ReceiptsSyncPhases::WaitingForValidate; + } + ReceiptsSyncPhases::WaitingForValidate => { + // This block now contains a chunk with the transaction sent above. + if let NetworkRequests::Block { block } = msg { + assert!(block.header().height() >= wait_till); + assert!(block.header().height() <= wait_till + 20); + if block.header().height() == wait_till + 20 { + System::current().stop(); + } + if block.header().height() == wait_till + 10 { + for i in 0..16 { + actix::spawn( + connectors1.write().unwrap()[i] + .1 + .send(Query::new( + BlockReference::latest(), + QueryRequest::ViewAccount { + account_id: account_to.clone(), + }, + )) + .then(move |res| { + let res_inner = res.unwrap(); + if let Ok(query_response) = res_inner { + if let ViewAccount(view_account_result) = + query_response.kind + { + assert_eq!( + view_account_result.amount, + 1111 + ); } - future::ready(()) - }), - ); - } + } + future::ready(()) + }), + ); } } } - }; - (NetworkResponses::NoResponse.into(), true) - }, - ))), - ); - *connectors.write().unwrap() = conn; - let mut max_wait_ms = 240000; - if sync_hold { - max_wait_ms *= TEST_STATE_SYNC_TIMEOUT as u64; - } - - near_network::test_utils::wait_or_panic(max_wait_ms); - }); - } - - enum RandomSinglePartPhases { - WaitingForFirstBlock, - WaitingForThirdEpoch, - WaitingForSixEpoch, - } - - /// Verifies that fetching of random parts works properly by issuing transactions during the - /// third epoch, and then making sure that the balances are correct for the next three epochs. - /// If random one parts fetched during the epoch preceding the epoch a block producer is - /// assigned to were to have incorrect receipts, the balances in the fourth epoch would have - /// been incorrect due to wrong receipts applied during the third epoch. - #[test] - fn test_catchup_random_single_part_sync() { - test_catchup_random_single_part_sync_common(false, false, 13) - } - - // Same test as `test_catchup_random_single_part_sync`, but skips the chunks on height 14 and 15 - // It causes all the receipts to be applied only on height 16, which is the next epoch. - // It tests that the incoming receipts are property synced through epochs - #[test] - fn test_catchup_random_single_part_sync_skip_15() { - test_catchup_random_single_part_sync_common(true, false, 13) - } - - #[test] - fn test_catchup_random_single_part_sync_send_15() { - test_catchup_random_single_part_sync_common(false, false, 15) - } - - // Make sure that transactions are at least applied. - #[test] - fn test_catchup_random_single_part_sync_non_zero_amounts() { - test_catchup_random_single_part_sync_common(false, true, 13) - } - - // Use another height to send txs. - #[test] - fn test_catchup_random_single_part_sync_height_6() { - test_catchup_random_single_part_sync_common(false, false, 6) - } - - fn test_catchup_random_single_part_sync_common(skip_15: bool, non_zero: bool, height: u64) { - let validator_groups = 2; - init_integration_logger(); - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - - let (validators, key_pairs) = get_validators_and_key_pairs(); - let flat_validators = validators.iter().flatten().cloned().collect::>(); - - let phase = Arc::new(RwLock::new(RandomSinglePartPhases::WaitingForFirstBlock)); - let seen_heights_same_block = Arc::new(RwLock::new(HashSet::::new())); - - let amounts = Arc::new(RwLock::new(HashMap::new())); - - let check_amount = move |amounts: Arc>>, - account_id: AccountId, - amount: u128| { + } + }; + (NetworkResponses::NoResponse.into(), true) + }, + ))), + ); + *connectors.write().unwrap() = conn; + let mut max_wait_ms = 240000; + if sync_hold { + max_wait_ms *= TEST_STATE_SYNC_TIMEOUT as u64; + } + + near_network::test_utils::wait_or_panic(max_wait_ms); + }); +} + +enum RandomSinglePartPhases { + WaitingForFirstBlock, + WaitingForThirdEpoch, + WaitingForSixEpoch, +} + +/// Verifies that fetching of random parts works properly by issuing transactions during the +/// third epoch, and then making sure that the balances are correct for the next three epochs. +/// If random one parts fetched during the epoch preceding the epoch a block producer is +/// assigned to were to have incorrect receipts, the balances in the fourth epoch would have +/// been incorrect due to wrong receipts applied during the third epoch. +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_random_single_part_sync() { + test_catchup_random_single_part_sync_common(false, false, 13) +} + +// Same test as `test_catchup_random_single_part_sync`, but skips the chunks on height 14 and 15 +// It causes all the receipts to be applied only on height 16, which is the next epoch. +// It tests that the incoming receipts are property synced through epochs +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_random_single_part_sync_skip_15() { + test_catchup_random_single_part_sync_common(true, false, 13) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_random_single_part_sync_send_15() { + test_catchup_random_single_part_sync_common(false, false, 15) +} + +// Make sure that transactions are at least applied. +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_random_single_part_sync_non_zero_amounts() { + test_catchup_random_single_part_sync_common(false, true, 13) +} + +// Use another height to send txs. +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_random_single_part_sync_height_6() { + test_catchup_random_single_part_sync_common(false, false, 6) +} + +fn test_catchup_random_single_part_sync_common(skip_15: bool, non_zero: bool, height: u64) { + let validator_groups = 2; + init_integration_logger(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); + + let (validators, key_pairs) = get_validators_and_key_pairs(); + let flat_validators = validators.iter().flatten().cloned().collect::>(); + + let phase = Arc::new(RwLock::new(RandomSinglePartPhases::WaitingForFirstBlock)); + let seen_heights_same_block = Arc::new(RwLock::new(HashSet::::new())); + + let amounts = Arc::new(RwLock::new(HashMap::new())); + + let check_amount = + move |amounts: Arc>>, account_id: AccountId, amount: u128| { match amounts.write().unwrap().entry(account_id.clone()) { Entry::Occupied(entry) => { println!("OCCUPIED {:?}", entry); @@ -446,558 +445,544 @@ mod tests { } }; - let connectors1 = connectors.clone(); - let (_, conn, _) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - validator_groups, - true, - 6000, - false, - false, - 5, - true, - vec![false; validators.iter().map(|x| x.len()).sum()], - vec![true; validators.iter().map(|x| x.len()).sum()], - false, - Arc::new(RwLock::new(Box::new( - move |_account_id: _, msg: &PeerManagerMessageRequest| { - let msg = msg.as_network_requests_ref(); - let mut seen_heights_same_block = seen_heights_same_block.write().unwrap(); - let mut phase = phase.write().unwrap(); - match *phase { - RandomSinglePartPhases::WaitingForFirstBlock => { - if let NetworkRequests::Block { block } = msg { - assert_eq!(block.header().height(), 1); - *phase = RandomSinglePartPhases::WaitingForThirdEpoch; - } + let connectors1 = connectors.clone(); + let (_, conn, _) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + validator_groups, + true, + 6000, + false, + false, + 5, + true, + vec![false; validators.iter().map(|x| x.len()).sum()], + vec![true; validators.iter().map(|x| x.len()).sum()], + false, + Arc::new(RwLock::new(Box::new( + move |_account_id: _, msg: &PeerManagerMessageRequest| { + let msg = msg.as_network_requests_ref(); + let mut seen_heights_same_block = seen_heights_same_block.write().unwrap(); + let mut phase = phase.write().unwrap(); + match *phase { + RandomSinglePartPhases::WaitingForFirstBlock => { + if let NetworkRequests::Block { block } = msg { + assert_eq!(block.header().height(), 1); + *phase = RandomSinglePartPhases::WaitingForThirdEpoch; } - RandomSinglePartPhases::WaitingForThirdEpoch => { - if let NetworkRequests::Block { block } = msg { - if block.header().height() == 1 { - return (NetworkResponses::NoResponse.into(), false); - } - assert!(block.header().height() >= 2); - assert!(block.header().height() <= height); - let mut tx_count = 0; - if block.header().height() == height - && block.header().height() >= 2 - { - for (i, validator1) in flat_validators.iter().enumerate() { - for (j, validator2) in - flat_validators.iter().enumerate() - { - let mut amount = - (((i + j + 17) * 701) % 42 + 1) as u128; - if non_zero { - if i > j { - amount = 2; - } else { - amount = 1; - } + } + RandomSinglePartPhases::WaitingForThirdEpoch => { + if let NetworkRequests::Block { block } = msg { + if block.header().height() == 1 { + return (NetworkResponses::NoResponse.into(), false); + } + assert!(block.header().height() >= 2); + assert!(block.header().height() <= height); + let mut tx_count = 0; + if block.header().height() == height && block.header().height() >= 2 + { + for (i, validator1) in flat_validators.iter().enumerate() { + for (j, validator2) in flat_validators.iter().enumerate() { + let mut amount = + (((i + j + 17) * 701) % 42 + 1) as u128; + if non_zero { + if i > j { + amount = 2; + } else { + amount = 1; } - println!( - "VALUES {:?} {:?} {:?}", - validator1.to_string(), - validator2.to_string(), - amount + } + println!( + "VALUES {:?} {:?} {:?}", + validator1.to_string(), + validator2.to_string(), + amount + ); + for conn in 0..flat_validators.len() { + send_tx( + &connectors1.write().unwrap()[conn].0, + validator1.clone(), + validator2.clone(), + amount, + (12345 + tx_count) as u64, + *block.header().prev_hash(), ); - for conn in 0..flat_validators.len() { - send_tx( - &connectors1.write().unwrap()[conn].0, - validator1.clone(), - validator2.clone(), - amount, - (12345 + tx_count) as u64, - *block.header().prev_hash(), - ); - } - tx_count += 1; } + tx_count += 1; } - *phase = RandomSinglePartPhases::WaitingForSixEpoch; - assert_eq!(tx_count, 16 * 16); } + *phase = RandomSinglePartPhases::WaitingForSixEpoch; + assert_eq!(tx_count, 16 * 16); } } - RandomSinglePartPhases::WaitingForSixEpoch => { - if let NetworkRequests::Block { block } = msg { - assert!(block.header().height() >= height); - assert!(block.header().height() <= 32); - let check_height = if skip_15 { 28 } else { 26 }; - if block.header().height() >= check_height { - println!("BLOCK HEIGHT {:?}", block.header().height()); - for i in 0..16 { - for j in 0..16 { - let amounts1 = amounts.clone(); - let validator = flat_validators[j].clone(); - actix::spawn( - connectors1.write().unwrap()[i] - .1 - .send(Query::new( - BlockReference::latest(), - QueryRequest::ViewAccount { - account_id: flat_validators[j] - .clone(), - }, - )) - .then(move |res| { - let res_inner = res.unwrap(); - if let Ok(query_response) = res_inner { - if let ViewAccount( - view_account_result, - ) = query_response.kind - { - check_amount( - amounts1, - validator, - view_account_result.amount, - ); - } + } + RandomSinglePartPhases::WaitingForSixEpoch => { + if let NetworkRequests::Block { block } = msg { + assert!(block.header().height() >= height); + assert!(block.header().height() <= 32); + let check_height = if skip_15 { 28 } else { 26 }; + if block.header().height() >= check_height { + println!("BLOCK HEIGHT {:?}", block.header().height()); + for i in 0..16 { + for j in 0..16 { + let amounts1 = amounts.clone(); + let validator = flat_validators[j].clone(); + actix::spawn( + connectors1.write().unwrap()[i] + .1 + .send(Query::new( + BlockReference::latest(), + QueryRequest::ViewAccount { + account_id: flat_validators[j].clone(), + }, + )) + .then(move |res| { + let res_inner = res.unwrap(); + if let Ok(query_response) = res_inner { + if let ViewAccount( + view_account_result, + ) = query_response.kind + { + check_amount( + amounts1, + validator, + view_account_result.amount, + ); } - future::ready(()) - }), - ); - } - } - } - if block.header().height() == 32 { - println!( - "SEEN HEIGHTS SAME BLOCK {:?}", - seen_heights_same_block.len() - ); - assert_eq!(seen_heights_same_block.len(), 1); - let amounts1 = amounts.clone(); - for flat_validator in &flat_validators { - match amounts1 - .write() - .unwrap() - .entry(flat_validator.clone()) - { - Entry::Occupied(_) => { - continue; - } - Entry::Vacant(entry) => { - println!( - "VALIDATOR = {:?}, ENTRY = {:?}", - flat_validator, entry - ); - assert!(false); - } - }; + } + future::ready(()) + }), + ); } - System::current().stop(); } } - if let NetworkRequests::PartialEncodedChunkMessage { - partial_encoded_chunk, - .. - } = msg - { - if partial_encoded_chunk.header.height_created() == 22 { - seen_heights_same_block - .insert(partial_encoded_chunk.header.prev_block_hash()); - } - if skip_15 { - if partial_encoded_chunk.header.height_created() == 14 - || partial_encoded_chunk.header.height_created() == 15 + if block.header().height() == 32 { + println!( + "SEEN HEIGHTS SAME BLOCK {:?}", + seen_heights_same_block.len() + ); + assert_eq!(seen_heights_same_block.len(), 1); + let amounts1 = amounts.clone(); + for flat_validator in &flat_validators { + match amounts1 + .write() + .unwrap() + .entry(flat_validator.clone()) { - return (NetworkResponses::NoResponse.into(), false); - } + Entry::Occupied(_) => { + continue; + } + Entry::Vacant(entry) => { + println!( + "VALIDATOR = {:?}, ENTRY = {:?}", + flat_validator, entry + ); + assert!(false); + } + }; } + System::current().stop(); } } - }; - (NetworkResponses::NoResponse.into(), true) - }, - ))), - ); - *connectors.write().unwrap() = conn; - - near_network::test_utils::wait_or_panic(480000); - }); - } - - /// Makes sure that 24 consecutive blocks are produced by 12 validators split into three epochs. - /// For extra coverage doesn't allow block propagation of some heights (and expects the blocks - /// to be skipped) - /// This test would fail if at any point validators got stuck with state sync, or block - /// production stalled for any other reason. - #[test] - fn test_catchup_sanity_blocks_produced() { - let validator_groups = 2; - init_integration_logger(); - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - - let heights = Arc::new(RwLock::new(HashMap::new())); - let heights1 = heights.clone(); - - let check_height = - move |hash: CryptoHash, height| match heights1.write().unwrap().entry(hash.clone()) - { - Entry::Occupied(entry) => { - assert_eq!(*entry.get(), height); - } - Entry::Vacant(entry) => { - entry.insert(height); - } - }; - - let (validators, key_pairs) = get_validators_and_key_pairs(); - - let (_, conn, _) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - validator_groups, - true, - 2000, - false, - false, - 5, - true, - vec![false; validators.iter().map(|x| x.len()).sum()], - vec![true; validators.iter().map(|x| x.len()).sum()], - false, - Arc::new(RwLock::new(Box::new( - move |_account_id: _, msg: &PeerManagerMessageRequest| { - let msg = msg.as_network_requests_ref(); - let propagate = if let NetworkRequests::Block { block } = msg { - check_height(*block.hash(), block.header().height()); - - if block.header().height() % 10 == 5 { - check_height( - *block.header().prev_hash(), - block.header().height() - 2, - ); - } else { - check_height( - *block.header().prev_hash(), - block.header().height() - 1, - ); + if let NetworkRequests::PartialEncodedChunkMessage { + partial_encoded_chunk, + .. + } = msg + { + if partial_encoded_chunk.header.height_created() == 22 { + seen_heights_same_block + .insert(partial_encoded_chunk.header.prev_block_hash()); + } + if skip_15 { + if partial_encoded_chunk.header.height_created() == 14 + || partial_encoded_chunk.header.height_created() == 15 + { + return (NetworkResponses::NoResponse.into(), false); + } + } } + } + }; + (NetworkResponses::NoResponse.into(), true) + }, + ))), + ); + *connectors.write().unwrap() = conn; + + near_network::test_utils::wait_or_panic(480000); + }); +} - if block.header().height() >= 25 { - System::current().stop(); - } +/// Makes sure that 24 consecutive blocks are produced by 12 validators split into three epochs. +/// For extra coverage doesn't allow block propagation of some heights (and expects the blocks +/// to be skipped) +/// This test would fail if at any point validators got stuck with state sync, or block +/// production stalled for any other reason. +#[cfg(feature = "expensive_tests")] +#[test] +fn test_catchup_sanity_blocks_produced() { + let validator_groups = 2; + init_integration_logger(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); + + let heights = Arc::new(RwLock::new(HashMap::new())); + let heights1 = heights.clone(); + + let check_height = + move |hash: CryptoHash, height| match heights1.write().unwrap().entry(hash.clone()) { + Entry::Occupied(entry) => { + assert_eq!(*entry.get(), height); + } + Entry::Vacant(entry) => { + entry.insert(height); + } + }; - // Do not propagate blocks at heights %10=4 - block.header().height() % 10 != 4 + let (validators, key_pairs) = get_validators_and_key_pairs(); + + let (_, conn, _) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + validator_groups, + true, + 2000, + false, + false, + 5, + true, + vec![false; validators.iter().map(|x| x.len()).sum()], + vec![true; validators.iter().map(|x| x.len()).sum()], + false, + Arc::new(RwLock::new(Box::new( + move |_account_id: _, msg: &PeerManagerMessageRequest| { + let msg = msg.as_network_requests_ref(); + let propagate = if let NetworkRequests::Block { block } = msg { + check_height(*block.hash(), block.header().height()); + + if block.header().height() % 10 == 5 { + check_height(*block.header().prev_hash(), block.header().height() - 2); } else { - true - }; + check_height(*block.header().prev_hash(), block.header().height() - 1); + } - (NetworkResponses::NoResponse.into(), propagate) - }, - ))), - ); - *connectors.write().unwrap() = conn; - - near_network::test_utils::wait_or_panic(60000); - }); - } - - enum ChunkGrievingPhases { - FirstAttack, - SecondAttack, - } - - // TODO(#3180): seals are disabled in single shard setting - #[test] - #[ignore] - fn test_chunk_grieving() { - let validator_groups = 1; - init_integration_logger(); - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - - let (validators, key_pairs) = get_validators_and_key_pairs(); - - let malicious_node = "test3.6".parse().unwrap(); - let victim_node = "test3.5".parse().unwrap(); - let phase = Arc::new(RwLock::new(ChunkGrievingPhases::FirstAttack)); - let grieving_chunk_hash = Arc::new(RwLock::new(ChunkHash::default())); - let unaccepted_block_hash = Arc::new(RwLock::new(CryptoHash::default())); - - let _connectors1 = connectors.clone(); - - let block_prod_time: u64 = 3500; - let (_, conn, _) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - validator_groups, - true, - block_prod_time, - false, - false, - 5, - true, - vec![false; validators.iter().map(|x| x.len()).sum()], - vec![true; validators.iter().map(|x| x.len()).sum()], - false, - Arc::new(RwLock::new(Box::new( - move |sender_account_id: AccountId, msg: &PeerManagerMessageRequest| { - let msg = msg.as_network_requests_ref(); - let mut grieving_chunk_hash = grieving_chunk_hash.write().unwrap(); - let mut unaccepted_block_hash = unaccepted_block_hash.write().unwrap(); - let mut phase = phase.write().unwrap(); - match *phase { - ChunkGrievingPhases::FirstAttack => { - if let NetworkRequests::PartialEncodedChunkMessage { - partial_encoded_chunk, - account_id, - } = msg - { - let height = partial_encoded_chunk.header.height_created(); - let shard_id = partial_encoded_chunk.header.shard_id(); - if height == 12 && shard_id == 0 { - // "test3.6" is the chunk producer on height 12, shard_id 0 - assert_eq!(sender_account_id, malicious_node); - println!( - "ACCOUNT {:?} PARTS {:?} CHUNK {:?}", - account_id, - partial_encoded_chunk.parts.len(), - partial_encoded_chunk - ); - if account_id == &victim_node { - // "test3.5" is a block producer of block on height 12, sending to it - *grieving_chunk_hash = - partial_encoded_chunk.header.chunk_hash(); - } else { - return (NetworkResponses::NoResponse.into(), false); - } + if block.header().height() >= 25 { + System::current().stop(); + } + + // Do not propagate blocks at heights %10=4 + block.header().height() % 10 != 4 + } else { + true + }; + + (NetworkResponses::NoResponse.into(), propagate) + }, + ))), + ); + *connectors.write().unwrap() = conn; + + near_network::test_utils::wait_or_panic(60000); + }); +} + +enum ChunkGrievingPhases { + FirstAttack, + SecondAttack, +} + +// TODO(#3180): seals are disabled in single shard setting +#[cfg(feature = "expensive_tests")] +#[test] +#[ignore] +fn test_chunk_grieving() { + let validator_groups = 1; + init_integration_logger(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); + + let (validators, key_pairs) = get_validators_and_key_pairs(); + + let malicious_node = "test3.6".parse().unwrap(); + let victim_node = "test3.5".parse().unwrap(); + let phase = Arc::new(RwLock::new(ChunkGrievingPhases::FirstAttack)); + let grieving_chunk_hash = Arc::new(RwLock::new(ChunkHash::default())); + let unaccepted_block_hash = Arc::new(RwLock::new(CryptoHash::default())); + + let _connectors1 = connectors.clone(); + + let block_prod_time: u64 = 3500; + let (_, conn, _) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + validator_groups, + true, + block_prod_time, + false, + false, + 5, + true, + vec![false; validators.iter().map(|x| x.len()).sum()], + vec![true; validators.iter().map(|x| x.len()).sum()], + false, + Arc::new(RwLock::new(Box::new( + move |sender_account_id: AccountId, msg: &PeerManagerMessageRequest| { + let msg = msg.as_network_requests_ref(); + let mut grieving_chunk_hash = grieving_chunk_hash.write().unwrap(); + let mut unaccepted_block_hash = unaccepted_block_hash.write().unwrap(); + let mut phase = phase.write().unwrap(); + match *phase { + ChunkGrievingPhases::FirstAttack => { + if let NetworkRequests::PartialEncodedChunkMessage { + partial_encoded_chunk, + account_id, + } = msg + { + let height = partial_encoded_chunk.header.height_created(); + let shard_id = partial_encoded_chunk.header.shard_id(); + if height == 12 && shard_id == 0 { + // "test3.6" is the chunk producer on height 12, shard_id 0 + assert_eq!(sender_account_id, malicious_node); + println!( + "ACCOUNT {:?} PARTS {:?} CHUNK {:?}", + account_id, + partial_encoded_chunk.parts.len(), + partial_encoded_chunk + ); + if account_id == &victim_node { + // "test3.5" is a block producer of block on height 12, sending to it + *grieving_chunk_hash = + partial_encoded_chunk.header.chunk_hash(); + } else { + return (NetworkResponses::NoResponse.into(), false); } } - if let NetworkRequests::Block { block } = msg { - if block.header().height() == 12 { - println!("BLOCK {:?}", block); - *unaccepted_block_hash = *block.header().hash(); - assert_eq!(4, block.header().chunks_included()); - *phase = ChunkGrievingPhases::SecondAttack; - } + } + if let NetworkRequests::Block { block } = msg { + if block.header().height() == 12 { + println!("BLOCK {:?}", block); + *unaccepted_block_hash = *block.header().hash(); + assert_eq!(4, block.header().chunks_included()); + *phase = ChunkGrievingPhases::SecondAttack; } } - ChunkGrievingPhases::SecondAttack => { - if let NetworkRequests::PartialEncodedChunkRequest { - request, - target: - AccountIdOrPeerTrackingShard { - account_id: Some(account_id), - .. - }, - } = msg - { - if request.chunk_hash == *grieving_chunk_hash { - if account_id == &malicious_node { - // holding grieving_chunk_hash by malicious node - return (NetworkResponses::NoResponse.into(), false); - } + } + ChunkGrievingPhases::SecondAttack => { + if let NetworkRequests::PartialEncodedChunkRequest { + request, + target: + AccountIdOrPeerTrackingShard { + account_id: Some(account_id), .. + }, + } = msg + { + if request.chunk_hash == *grieving_chunk_hash { + if account_id == &malicious_node { + // holding grieving_chunk_hash by malicious node + return (NetworkResponses::NoResponse.into(), false); } - } else if let NetworkRequests::PartialEncodedChunkRequest { - request: _, - target: _, - } = msg - { - // this test was written before the feature that allows - // sending requests directly to the peer. The test likely never - // triggers this path, but if this assert triggers, the above - // `if let` needs to be extended to block messages sent to the - // malicious node directly via the peer id - assert!(false); } - if let NetworkRequests::PartialEncodedChunkResponse { - route_back: _, - response, - } = msg - { - if response.chunk_hash == *grieving_chunk_hash { - // Only victim_node knows some parts of grieving_chunk_hash - // It's not enough to restore the chunk completely - assert_eq!(sender_account_id, victim_node); - } + } else if let NetworkRequests::PartialEncodedChunkRequest { + request: _, + target: _, + } = msg + { + // this test was written before the feature that allows + // sending requests directly to the peer. The test likely never + // triggers this path, but if this assert triggers, the above + // `if let` needs to be extended to block messages sent to the + // malicious node directly via the peer id + assert!(false); + } + if let NetworkRequests::PartialEncodedChunkResponse { + route_back: _, + response, + } = msg + { + if response.chunk_hash == *grieving_chunk_hash { + // Only victim_node knows some parts of grieving_chunk_hash + // It's not enough to restore the chunk completely + assert_eq!(sender_account_id, victim_node); } - if let NetworkRequests::PartialEncodedChunkMessage { - partial_encoded_chunk, - account_id, - } = msg - { - let height = partial_encoded_chunk.header.height_created(); - let shard_id = partial_encoded_chunk.header.shard_id(); - if height == 42 && shard_id == 2 { - // "test3.6" is the chunk producer on height 42, shard_id 2 - assert_eq!(sender_account_id, malicious_node); - println!( - "ACCOUNT {:?} PARTS {:?} CHUNK {:?}", - account_id, - partial_encoded_chunk.parts.len(), - partial_encoded_chunk - ); - } + } + if let NetworkRequests::PartialEncodedChunkMessage { + partial_encoded_chunk, + account_id, + } = msg + { + let height = partial_encoded_chunk.header.height_created(); + let shard_id = partial_encoded_chunk.header.shard_id(); + if height == 42 && shard_id == 2 { + // "test3.6" is the chunk producer on height 42, shard_id 2 + assert_eq!(sender_account_id, malicious_node); + println!( + "ACCOUNT {:?} PARTS {:?} CHUNK {:?}", + account_id, + partial_encoded_chunk.parts.len(), + partial_encoded_chunk + ); } - if let NetworkRequests::Block { block } = msg { - if block.header().height() == 42 { - println!("BLOCK {:?}", block,); - // This is the main assert of the test - // Chunk from malicious node shouldn't be accepted at all - assert_eq!(3, block.header().chunks_included()); - System::current().stop(); - } + } + if let NetworkRequests::Block { block } = msg { + if block.header().height() == 42 { + println!("BLOCK {:?}", block,); + // This is the main assert of the test + // Chunk from malicious node shouldn't be accepted at all + assert_eq!(3, block.header().chunks_included()); + System::current().stop(); } } + } + }; + (NetworkResponses::NoResponse.into(), true) + }, + ))), + ); + *connectors.write().unwrap() = conn; + let max_wait_ms = 240000; + + near_network::test_utils::wait_or_panic(max_wait_ms); + }); +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_all_chunks_accepted_1000() { + test_all_chunks_accepted_common(1000, 3000, 5) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_all_chunks_accepted_1000_slow() { + test_all_chunks_accepted_common(1000, 6000, 5) +} + +#[cfg(feature = "expensive_tests")] +#[test] +fn test_all_chunks_accepted_1000_rare_epoch_changing() { + test_all_chunks_accepted_common(1000, 1500, 100) +} + +fn test_all_chunks_accepted_common( + last_height: BlockHeight, + block_prod_time: u64, + epoch_length: BlockHeightDelta, +) { + let validator_groups = 1; + init_integration_logger(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); + + let (validators, key_pairs) = get_validators_and_key_pairs(); + let verbose = false; + + let _connectors1 = connectors.clone(); + let seen_chunk_same_sender = Arc::new(RwLock::new(HashSet::<(AccountId, u64, u64)>::new())); + let requested = Arc::new(RwLock::new(HashSet::<(AccountId, Vec, ChunkHash)>::new())); + let responded = Arc::new(RwLock::new(HashSet::<(CryptoHash, Vec, ChunkHash)>::new())); + + let (_, conn, _) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + validator_groups, + true, + block_prod_time, + false, + false, + epoch_length, + true, + vec![false; validators.iter().map(|x| x.len()).sum()], + vec![true; validators.iter().map(|x| x.len()).sum()], + false, + Arc::new(RwLock::new(Box::new( + move |sender_account_id: AccountId, msg: &PeerManagerMessageRequest| { + let msg = msg.as_network_requests_ref(); + let mut seen_chunk_same_sender = seen_chunk_same_sender.write().unwrap(); + let mut requested = requested.write().unwrap(); + let mut responded = responded.write().unwrap(); + if let NetworkRequests::PartialEncodedChunkMessage { + account_id, + partial_encoded_chunk, + } = msg + { + let header = &partial_encoded_chunk.header; + if seen_chunk_same_sender.contains(&( + account_id.clone(), + header.height_created(), + header.shard_id(), + )) { + println!("=== SAME CHUNK AGAIN!"); + assert!(false); }; - (NetworkResponses::NoResponse.into(), true) - }, - ))), - ); - *connectors.write().unwrap() = conn; - let max_wait_ms = 240000; - - near_network::test_utils::wait_or_panic(max_wait_ms); - }); - } - - #[test] - fn test_all_chunks_accepted_1000() { - test_all_chunks_accepted_common(1000, 3000, 5) - } - - #[test] - fn test_all_chunks_accepted_1000_slow() { - test_all_chunks_accepted_common(1000, 6000, 5) - } - - #[test] - fn test_all_chunks_accepted_1000_rare_epoch_changing() { - test_all_chunks_accepted_common(1000, 1500, 100) - } - - fn test_all_chunks_accepted_common( - last_height: BlockHeight, - block_prod_time: u64, - epoch_length: BlockHeightDelta, - ) { - let validator_groups = 1; - init_integration_logger(); - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - - let (validators, key_pairs) = get_validators_and_key_pairs(); - let verbose = false; - - let _connectors1 = connectors.clone(); - let seen_chunk_same_sender = - Arc::new(RwLock::new(HashSet::<(AccountId, u64, u64)>::new())); - let requested = - Arc::new(RwLock::new(HashSet::<(AccountId, Vec, ChunkHash)>::new())); - let responded = - Arc::new(RwLock::new(HashSet::<(CryptoHash, Vec, ChunkHash)>::new())); - - let (_, conn, _) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - validator_groups, - true, - block_prod_time, - false, - false, - epoch_length, - true, - vec![false; validators.iter().map(|x| x.len()).sum()], - vec![true; validators.iter().map(|x| x.len()).sum()], - false, - Arc::new(RwLock::new(Box::new( - move |sender_account_id: AccountId, msg: &PeerManagerMessageRequest| { - let msg = msg.as_network_requests_ref(); - let mut seen_chunk_same_sender = seen_chunk_same_sender.write().unwrap(); - let mut requested = requested.write().unwrap(); - let mut responded = responded.write().unwrap(); - if let NetworkRequests::PartialEncodedChunkMessage { - account_id, - partial_encoded_chunk, - } = msg - { - let header = &partial_encoded_chunk.header; - if seen_chunk_same_sender.contains(&( - account_id.clone(), - header.height_created(), - header.shard_id(), + seen_chunk_same_sender.insert(( + account_id.clone(), + header.height_created(), + header.shard_id(), + )); + } + if let NetworkRequests::PartialEncodedChunkRequest { target: _, request } = msg + { + if verbose { + if requested.contains(&( + sender_account_id.clone(), + request.part_ords.clone(), + request.chunk_hash.clone(), )) { - println!("=== SAME CHUNK AGAIN!"); - assert!(false); + println!("=== SAME REQUEST AGAIN!"); }; - seen_chunk_same_sender.insert(( - account_id.clone(), - header.height_created(), - header.shard_id(), + requested.insert(( + sender_account_id, + request.part_ords.clone(), + request.chunk_hash.clone(), )); } - if let NetworkRequests::PartialEncodedChunkRequest { target: _, request } = - msg - { - if verbose { - if requested.contains(&( - sender_account_id.clone(), - request.part_ords.clone(), - request.chunk_hash.clone(), - )) { - println!("=== SAME REQUEST AGAIN!"); - }; - requested.insert(( - sender_account_id, - request.part_ords.clone(), - request.chunk_hash.clone(), - )); + } + if let NetworkRequests::PartialEncodedChunkResponse { route_back, response } = + msg + { + if verbose { + if responded.contains(&( + route_back.clone(), + response.parts.iter().map(|x| x.part_ord).collect(), + response.chunk_hash.clone(), + )) { + println!("=== SAME RESPONSE AGAIN!"); } + responded.insert(( + route_back.clone(), + response.parts.iter().map(|x| x.part_ord).collect(), + response.chunk_hash.clone(), + )); } - if let NetworkRequests::PartialEncodedChunkResponse { - route_back, - response, - } = msg - { - if verbose { - if responded.contains(&( - route_back.clone(), - response.parts.iter().map(|x| x.part_ord).collect(), - response.chunk_hash.clone(), - )) { - println!("=== SAME RESPONSE AGAIN!"); + } + if let NetworkRequests::Block { block } = msg { + // There is no chunks at height 1 + if block.header().height() > 1 { + if block.header().height() % epoch_length != 1 { + if block.header().chunks_included() != 4 { + println!( + "BLOCK WITH {:?} CHUNKS, {:?}", + block.header().chunks_included(), + block + ); + assert!(false); } - responded.insert(( - route_back.clone(), - response.parts.iter().map(|x| x.part_ord).collect(), - response.chunk_hash.clone(), - )); } - } - if let NetworkRequests::Block { block } = msg { - // There is no chunks at height 1 - if block.header().height() > 1 { - if block.header().height() % epoch_length != 1 { - if block.header().chunks_included() != 4 { - println!( - "BLOCK WITH {:?} CHUNKS, {:?}", - block.header().chunks_included(), - block - ); - assert!(false); - } - } - if block.header().height() == last_height { - System::current().stop(); - } + if block.header().height() == last_height { + System::current().stop(); } } - (NetworkResponses::NoResponse.into(), true) - }, - ))), - ); - *connectors.write().unwrap() = conn; - let max_wait_ms = block_prod_time * last_height / 10 * 18 + 20000; - - near_network::test_utils::wait_or_panic(max_wait_ms); - }); - } + } + (NetworkResponses::NoResponse.into(), true) + }, + ))), + ); + *connectors.write().unwrap() = conn; + let max_wait_ms = block_prod_time * last_height / 10 * 18 + 20000; + + near_network::test_utils::wait_or_panic(max_wait_ms); + }); } diff --git a/chain/client/src/tests/consensus.rs b/chain/client/src/tests/consensus.rs index e48bcac2a36..152a3908611 100644 --- a/chain/client/src/tests/consensus.rs +++ b/chain/client/src/tests/consensus.rs @@ -1,298 +1,289 @@ -#[cfg(test)] -#[cfg(feature = "expensive_tests")] -mod tests { - use std::collections::{BTreeMap, HashMap, HashSet}; - use std::sync::{Arc, RwLock, RwLockWriteGuard}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::{Arc, RwLock, RwLockWriteGuard}; - use actix::{Addr, System}; - use rand::{thread_rng, Rng}; +use actix::{Addr, System}; +use rand::{thread_rng, Rng}; - use crate::test_utils::setup_mock_all_validators; - use crate::{ClientActor, ViewClientActor}; - use near_actix_test_utils::run_actix; - use near_chain::Block; - use near_logger_utils::init_integration_logger; - use near_network::types::{ - NetworkClientMessages, NetworkRequests, NetworkResponses, PeerManagerMessageRequest, - }; - use near_network_primitives::types::PeerInfo; - use near_primitives::block::{Approval, ApprovalInner}; - use near_primitives::types::{AccountId, BlockHeight}; +use crate::test_utils::setup_mock_all_validators; +use crate::{ClientActor, ViewClientActor}; +use near_actix_test_utils::run_actix; +use near_chain::Block; +use near_logger_utils::init_integration_logger; +use near_network::types::{ + NetworkClientMessages, NetworkRequests, NetworkResponses, PeerManagerMessageRequest, +}; +use near_network_primitives::types::PeerInfo; +use near_primitives::block::{Approval, ApprovalInner}; +use near_primitives::types::{AccountId, BlockHeight}; - /// Rotates three independent sets of block producers producing blocks with a very short epoch length. - /// Occasionally when an endorsement comes, make all the endorsers send a skip message far-ish into - /// the future, and delay the distribution of the block produced this way. - /// Periodically verify finality is not violated. - /// This test is designed to reproduce finality bugs on the epoch boundaries. - #[test] - fn test_consensus_with_epoch_switches() { - init_integration_logger(); +/// Rotates three independent sets of block producers producing blocks with a very short epoch length. +/// Occasionally when an endorsement comes, make all the endorsers send a skip message far-ish into +/// the future, and delay the distribution of the block produced this way. +/// Periodically verify finality is not violated. +/// This test is designed to reproduce finality bugs on the epoch boundaries. +#[cfg(feature = "expensive_tests")] +#[test] +fn test_consensus_with_epoch_switches() { + init_integration_logger(); - const HEIGHT_GOAL: u64 = 120; + const HEIGHT_GOAL: u64 = 120; - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - let connectors1 = connectors.clone(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); + let connectors1 = connectors.clone(); - let validators: Vec> = [ - [ - "test1.1", "test1.2", "test1.3", "test1.4", "test1.5", "test1.6", "test1.7", - "test1.8", - ], - [ - "test2.1", "test2.2", "test2.3", "test2.4", "test2.5", "test2.6", "test2.7", - "test2.8", - ], - [ - "test3.1", "test3.2", "test3.3", "test3.4", "test3.5", "test3.6", "test3.7", - "test3.8", - ], - ] - .iter() - .map(|l| l.iter().map(|account_id| account_id.parse().unwrap()).collect()) - .collect(); - let key_pairs = (0..24).map(|_| PeerInfo::random()).collect::>(); + let validators: Vec> = [ + [ + "test1.1", "test1.2", "test1.3", "test1.4", "test1.5", "test1.6", "test1.7", + "test1.8", + ], + [ + "test2.1", "test2.2", "test2.3", "test2.4", "test2.5", "test2.6", "test2.7", + "test2.8", + ], + [ + "test3.1", "test3.2", "test3.3", "test3.4", "test3.5", "test3.6", "test3.7", + "test3.8", + ], + ] + .iter() + .map(|l| l.iter().map(|account_id| account_id.parse().unwrap()).collect()) + .collect(); + let key_pairs = (0..24).map(|_| PeerInfo::random()).collect::>(); - let block_to_prev_block = Arc::new(RwLock::new(HashMap::new())); - let block_to_height = Arc::new(RwLock::new(HashMap::new())); + let block_to_prev_block = Arc::new(RwLock::new(HashMap::new())); + let block_to_height = Arc::new(RwLock::new(HashMap::new())); - let all_blocks = Arc::new(RwLock::new(BTreeMap::new())); - let final_block_heights = Arc::new(RwLock::new(HashSet::new())); + let all_blocks = Arc::new(RwLock::new(BTreeMap::new())); + let final_block_heights = Arc::new(RwLock::new(HashSet::new())); - let largest_target_height = Arc::new(RwLock::new(vec![0u64; 24])); - let skips_per_height = Arc::new(RwLock::new(vec![])); + let largest_target_height = Arc::new(RwLock::new(vec![0u64; 24])); + let skips_per_height = Arc::new(RwLock::new(vec![])); - let largest_block_height = Arc::new(RwLock::new(0u64)); - let delayed_blocks = Arc::new(RwLock::new(vec![])); + let largest_block_height = Arc::new(RwLock::new(0u64)); + let delayed_blocks = Arc::new(RwLock::new(vec![])); - let (_, conn, _) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - 1, - true, - 1000, - false, - false, - 4, - true, - vec![true; validators.iter().map(|x| x.len()).sum()], - vec![false; validators.iter().map(|x| x.len()).sum()], - false, - Arc::new(RwLock::new(Box::new( - move |from_whom: AccountId, msg: &PeerManagerMessageRequest| { - let mut all_blocks: RwLockWriteGuard> = - all_blocks.write().unwrap(); - let mut final_block_heights = final_block_heights.write().unwrap(); - let mut block_to_height = block_to_height.write().unwrap(); - let mut block_to_prev_block = block_to_prev_block.write().unwrap(); - let mut largest_target_height = largest_target_height.write().unwrap(); - let mut skips_per_height = skips_per_height.write().unwrap(); - let mut largest_block_height = largest_block_height.write().unwrap(); + let (_, conn, _) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + 1, + true, + 1000, + false, + false, + 4, + true, + vec![true; validators.iter().map(|x| x.len()).sum()], + vec![false; validators.iter().map(|x| x.len()).sum()], + false, + Arc::new(RwLock::new(Box::new( + move |from_whom: AccountId, msg: &PeerManagerMessageRequest| { + let mut all_blocks: RwLockWriteGuard> = + all_blocks.write().unwrap(); + let mut final_block_heights = final_block_heights.write().unwrap(); + let mut block_to_height = block_to_height.write().unwrap(); + let mut block_to_prev_block = block_to_prev_block.write().unwrap(); + let mut largest_target_height = largest_target_height.write().unwrap(); + let mut skips_per_height = skips_per_height.write().unwrap(); + let mut largest_block_height = largest_block_height.write().unwrap(); - let mut delayed_blocks = delayed_blocks.write().unwrap(); + let mut delayed_blocks = delayed_blocks.write().unwrap(); - match msg.as_network_requests_ref() { - NetworkRequests::Block { block } => { - if !all_blocks.contains_key(&block.header().height()) { - println!( - "BLOCK @{} EPOCH: {:?}, APPROVALS: {:?}", - block.header().height(), - block.header().epoch_id(), - block - .header() - .approvals() - .iter() - .map(|x| if x.is_some() { 1 } else { 0 }) - .collect::>() - ); - } - all_blocks.insert(block.header().height(), block.clone()); - block_to_prev_block - .insert(*block.hash(), *block.header().prev_hash()); - block_to_height.insert(*block.hash(), block.header().height()); + match msg.as_network_requests_ref() { + NetworkRequests::Block { block } => { + if !all_blocks.contains_key(&block.header().height()) { + println!( + "BLOCK @{} EPOCH: {:?}, APPROVALS: {:?}", + block.header().height(), + block.header().epoch_id(), + block + .header() + .approvals() + .iter() + .map(|x| if x.is_some() { 1 } else { 0 }) + .collect::>() + ); + } + all_blocks.insert(block.header().height(), block.clone()); + block_to_prev_block.insert(*block.hash(), *block.header().prev_hash()); + block_to_height.insert(*block.hash(), block.header().height()); - if *largest_block_height / 20 < block.header().height() / 20 { - // Periodically verify the finality - println!("VERIFYING FINALITY CONDITIONS"); - for block in all_blocks.values() { - if let Some(prev_hash) = - block_to_prev_block.get(&block.hash()) - { - if let Some(prev_height) = - block_to_height.get(prev_hash) - { - let cur_height = block.header().height(); - for f in final_block_heights.iter() { - if f < &cur_height && f > prev_height { - assert!( - false, - "{} < {} < {}", - prev_height, f, cur_height - ); - } + if *largest_block_height / 20 < block.header().height() / 20 { + // Periodically verify the finality + println!("VERIFYING FINALITY CONDITIONS"); + for block in all_blocks.values() { + if let Some(prev_hash) = block_to_prev_block.get(&block.hash()) + { + if let Some(prev_height) = block_to_height.get(prev_hash) { + let cur_height = block.header().height(); + for f in final_block_heights.iter() { + if f < &cur_height && f > prev_height { + assert!( + false, + "{} < {} < {}", + prev_height, f, cur_height + ); } } } } - - if *largest_block_height >= HEIGHT_GOAL { - System::current().stop(); - } } - if block.header().height() > *largest_block_height + 3 { - *largest_block_height = block.header().height(); - if delayed_blocks.len() < 2 { - delayed_blocks.push(block.clone()); - return (NetworkResponses::NoResponse.into(), false); - } + if *largest_block_height >= HEIGHT_GOAL { + System::current().stop(); } - *largest_block_height = - std::cmp::max(block.header().height(), *largest_block_height); + } - let mut new_delayed_blocks = vec![]; - for delayed_block in delayed_blocks.iter() { - if delayed_block.hash() == block.hash() { - return (NetworkResponses::NoResponse.into(), false); - } - if delayed_block.header().height() - <= block.header().height() + 2 - { - for target_ord in 0..24 { - connectors1.write().unwrap()[target_ord].0.do_send( - NetworkClientMessages::Block( - delayed_block.clone(), - key_pairs[0].clone().id, - true, - ), - ); - } - } else { - new_delayed_blocks.push(delayed_block.clone()) - } + if block.header().height() > *largest_block_height + 3 { + *largest_block_height = block.header().height(); + if delayed_blocks.len() < 2 { + delayed_blocks.push(block.clone()); + return (NetworkResponses::NoResponse.into(), false); } - *delayed_blocks = new_delayed_blocks; + } + *largest_block_height = + std::cmp::max(block.header().height(), *largest_block_height); - let mut heights = vec![]; - let mut cur_hash = *block.hash(); - while let Some(height) = block_to_height.get(&cur_hash) { - heights.push(height); - cur_hash = block_to_prev_block.get(&cur_hash).unwrap().clone(); - if heights.len() > 10 { - break; + let mut new_delayed_blocks = vec![]; + for delayed_block in delayed_blocks.iter() { + if delayed_block.hash() == block.hash() { + return (NetworkResponses::NoResponse.into(), false); + } + if delayed_block.header().height() <= block.header().height() + 2 { + for target_ord in 0..24 { + connectors1.write().unwrap()[target_ord].0.do_send( + NetworkClientMessages::Block( + delayed_block.clone(), + key_pairs[0].clone().id, + true, + ), + ); } + } else { + new_delayed_blocks.push(delayed_block.clone()) } - // Use Doomslug finality, since without duplicate blocks at the same height - // it also provides safety under 1/3 faults - let is_final = heights.len() > 1 && heights[1] + 1 == *heights[0]; - println!( - "IS_FINAL: {} DELAYED: ({:?}) BLOCK: {} HISTORY: {:?}", - is_final, - delayed_blocks - .iter() - .map(|x| x.header().height()) - .collect::>(), - block.hash(), - heights, - ); + } + *delayed_blocks = new_delayed_blocks; - if is_final { - final_block_heights.insert(*heights[1]); + let mut heights = vec![]; + let mut cur_hash = *block.hash(); + while let Some(height) = block_to_height.get(&cur_hash) { + heights.push(height); + cur_hash = block_to_prev_block.get(&cur_hash).unwrap().clone(); + if heights.len() > 10 { + break; } } - NetworkRequests::Approval { approval_message } => { - // Identify who we are, and whom we are sending this message to - let mut epoch_id = 100; - let mut destination_ord = 100; - let mut my_ord = 100; + // Use Doomslug finality, since without duplicate blocks at the same height + // it also provides safety under 1/3 faults + let is_final = heights.len() > 1 && heights[1] + 1 == *heights[0]; + println!( + "IS_FINAL: {} DELAYED: ({:?}) BLOCK: {} HISTORY: {:?}", + is_final, + delayed_blocks + .iter() + .map(|x| x.header().height()) + .collect::>(), + block.hash(), + heights, + ); - for i in 0..validators.len() { - for j in 0..validators[i].len() { - if validators[i][j] == approval_message.target { - epoch_id = i; - destination_ord = j; - } - if validators[i][j] == from_whom { - my_ord = i * 8 + j; - } + if is_final { + final_block_heights.insert(*heights[1]); + } + } + NetworkRequests::Approval { approval_message } => { + // Identify who we are, and whom we are sending this message to + let mut epoch_id = 100; + let mut destination_ord = 100; + let mut my_ord = 100; + + for i in 0..validators.len() { + for j in 0..validators[i].len() { + if validators[i][j] == approval_message.target { + epoch_id = i; + destination_ord = j; + } + if validators[i][j] == from_whom { + my_ord = i * 8 + j; } } - assert_ne!(epoch_id, 100); - assert_ne!(my_ord, 100); - - // For each height we define `skips_per_height`, and each block producer sends - // skips that far into the future from that source height. - let source_height = match approval_message.approval.inner { - ApprovalInner::Endorsement(_) => { - if largest_target_height[my_ord] - >= approval_message.approval.target_height - && my_ord % 8 >= 2 - { - // We already manually sent a skip conflicting with this endorsement - // my_ord % 8 < 2 are two malicious actors in every epoch and they - // continue sending endorsements - return (NetworkResponses::NoResponse.into(), false); - } + } + assert_ne!(epoch_id, 100); + assert_ne!(my_ord, 100); - approval_message.approval.target_height - 1 + // For each height we define `skips_per_height`, and each block producer sends + // skips that far into the future from that source height. + let source_height = match approval_message.approval.inner { + ApprovalInner::Endorsement(_) => { + if largest_target_height[my_ord] + >= approval_message.approval.target_height + && my_ord % 8 >= 2 + { + // We already manually sent a skip conflicting with this endorsement + // my_ord % 8 < 2 are two malicious actors in every epoch and they + // continue sending endorsements + return (NetworkResponses::NoResponse.into(), false); } - ApprovalInner::Skip(source_height) => source_height, - }; - while source_height as usize >= skips_per_height.len() { - skips_per_height.push(if thread_rng().gen_bool(0.8) { - 0 - } else { - thread_rng().gen_range(2, 9) - }); + approval_message.approval.target_height - 1 } - if skips_per_height[source_height as usize] > 0 - && approval_message.approval.target_height - source_height == 1 - { - let delta = skips_per_height[source_height as usize]; - let approval = Approval { - target_height: approval_message.approval.target_height - + delta as u64, - inner: ApprovalInner::Skip(source_height), - ..approval_message.approval.clone() - }; - largest_target_height[my_ord] = std::cmp::max( - largest_target_height[my_ord], - approval.target_height as u64, - ); - connectors1.write().unwrap() - [epoch_id * 8 + (destination_ord + delta) % 8] - .0 - .do_send(NetworkClientMessages::BlockApproval( - approval, - key_pairs[my_ord].id.clone(), - )); - // Do not send the endorsement for couple block producers in each epoch - // This is needed because otherwise the block with enough endorsements - // sometimes comes faster than the sufficient number of skips is created, - // (because the block producer themselves doesn't send the endorsement - // over the network, they have one more approval ready to produce their - // block than the block producer that will be at the later height). If - // such a block is indeed produced faster than all the skips are created, - // the paritcipants who haven't sent their endorsements to be converted - // to skips change their head. - if my_ord % 8 < 2 { - return (NetworkResponses::NoResponse.into(), false); - } + ApprovalInner::Skip(source_height) => source_height, + }; + + while source_height as usize >= skips_per_height.len() { + skips_per_height.push(if thread_rng().gen_bool(0.8) { + 0 + } else { + thread_rng().gen_range(2, 9) + }); + } + if skips_per_height[source_height as usize] > 0 + && approval_message.approval.target_height - source_height == 1 + { + let delta = skips_per_height[source_height as usize]; + let approval = Approval { + target_height: approval_message.approval.target_height + + delta as u64, + inner: ApprovalInner::Skip(source_height), + ..approval_message.approval.clone() + }; + largest_target_height[my_ord] = std::cmp::max( + largest_target_height[my_ord], + approval.target_height as u64, + ); + connectors1.write().unwrap() + [epoch_id * 8 + (destination_ord + delta) % 8] + .0 + .do_send(NetworkClientMessages::BlockApproval( + approval, + key_pairs[my_ord].id.clone(), + )); + // Do not send the endorsement for couple block producers in each epoch + // This is needed because otherwise the block with enough endorsements + // sometimes comes faster than the sufficient number of skips is created, + // (because the block producer themselves doesn't send the endorsement + // over the network, they have one more approval ready to produce their + // block than the block producer that will be at the later height). If + // such a block is indeed produced faster than all the skips are created, + // the paritcipants who haven't sent their endorsements to be converted + // to skips change their head. + if my_ord % 8 < 2 { + return (NetworkResponses::NoResponse.into(), false); } } - _ => {} - }; - (NetworkResponses::NoResponse.into(), true) - }, - ))), - ); - *connectors.write().unwrap() = conn; + } + _ => {} + }; + (NetworkResponses::NoResponse.into(), true) + }, + ))), + ); + *connectors.write().unwrap() = conn; - // We only check the terminating condition once every 20 heights, thus extra 80 to - // account for possibly going beyond the HEIGHT_GOAL. - near_network::test_utils::wait_or_panic(3000 * (80 + HEIGHT_GOAL)); - }); - } + // We only check the terminating condition once every 20 heights, thus extra 80 to + // account for possibly going beyond the HEIGHT_GOAL. + near_network::test_utils::wait_or_panic(3000 * (80 + HEIGHT_GOAL)); + }); } diff --git a/chain/client/src/tests/cross_shard_tx.rs b/chain/client/src/tests/cross_shard_tx.rs index dbfc9dc45c6..1e95c848e7a 100644 --- a/chain/client/src/tests/cross_shard_tx.rs +++ b/chain/client/src/tests/cross_shard_tx.rs @@ -1,18 +1,29 @@ +#![allow(unused_imports)] + +use std::collections::HashSet; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; -use actix::{Addr, System}; +use actix::{Addr, MailboxError, System}; use futures::{future, FutureExt}; -use crate::test_utils::setup_mock_all_validators; -use crate::{ClientActor, Query, ViewClientActor}; use near_actix_test_utils::run_actix; +use near_chain::test_utils::account_id_to_shard_id; +use near_crypto::{InMemorySigner, KeyType}; use near_logger_utils::init_integration_logger; -use near_network::types::{NetworkResponses, PeerManagerMessageRequest}; +use near_network::types::{ + NetworkClientMessages, NetworkClientResponses, NetworkResponses, PeerManagerMessageRequest, + PeerManagerMessageResponse, +}; use near_network_primitives::types::PeerInfo; -use near_primitives::types::BlockReference; -use near_primitives::views::QueryRequest; +use near_primitives::hash::CryptoHash; +use near_primitives::transaction::SignedTransaction; +use near_primitives::types::{AccountId, BlockReference}; use near_primitives::views::QueryResponseKind::ViewAccount; +use near_primitives::views::{QueryRequest, QueryResponse}; + +use crate::test_utils::{setup_mock_all_validators, BlockStats}; +use crate::{ClientActor, Query, ViewClientActor}; /// Tests that the KeyValueRuntime properly sets balances in genesis and makes them queriable #[test] @@ -86,482 +97,457 @@ fn test_keyvalue_runtime_balances() { } #[cfg(feature = "expensive_tests")] -#[cfg(test)] -mod tests { - use std::collections::HashSet; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, RwLock}; - - use actix::{Addr, MailboxError, System}; - use futures::{future, FutureExt}; - - use crate::test_utils::{setup_mock_all_validators, BlockStats}; - use crate::{ClientActor, Query, ViewClientActor}; - use near_actix_test_utils::run_actix; - use near_chain::test_utils::account_id_to_shard_id; - use near_crypto::{InMemorySigner, KeyType}; - use near_logger_utils::init_integration_logger; - use near_network::types::{ - NetworkClientMessages, NetworkClientResponses, NetworkResponses, PeerManagerMessageRequest, - PeerManagerMessageResponse, - }; - use near_network_primitives::types::PeerInfo; - use near_primitives::hash::CryptoHash; - use near_primitives::transaction::SignedTransaction; - use near_primitives::types::{AccountId, BlockReference}; - use near_primitives::views::QueryResponseKind::ViewAccount; - use near_primitives::views::{QueryRequest, QueryResponse}; - - fn send_tx( - num_validators: usize, - connectors: Arc, Addr)>>>, - connector_ordinal: usize, - from: AccountId, - to: AccountId, - amount: u128, - nonce: u64, - block_hash: CryptoHash, - ) { - let connectors1 = connectors.clone(); - let signer = InMemorySigner::from_seed(from.clone(), KeyType::ED25519, from.as_ref()); - actix::spawn( - connectors.write().unwrap()[connector_ordinal] - .0 - .send(NetworkClientMessages::Transaction { - transaction: SignedTransaction::send_money( - nonce, - from.clone(), - to.clone(), - &signer, - amount, - block_hash, - ), - is_forwarded: false, - check_only: false, - }) - .then(move |x| { - match x.unwrap() { - NetworkClientResponses::NoResponse - | NetworkClientResponses::RequestRouted => { - assert_eq!(num_validators, 24); - send_tx( - num_validators, - connectors1, - (connector_ordinal + 8) % num_validators, - from, - to, - amount, - nonce, - block_hash, - ); - } - NetworkClientResponses::ValidTx => { - println!( - "Transaction was received by validator {:?}", - connector_ordinal - ); - } - other @ _ => { - println!( - "Transaction was rejected with an unexpected outcome: {:?}", - other - ); - assert!(false) - } +fn send_tx( + num_validators: usize, + connectors: Arc, Addr)>>>, + connector_ordinal: usize, + from: AccountId, + to: AccountId, + amount: u128, + nonce: u64, + block_hash: CryptoHash, +) { + let connectors1 = connectors.clone(); + let signer = InMemorySigner::from_seed(from.clone(), KeyType::ED25519, from.as_ref()); + actix::spawn( + connectors.write().unwrap()[connector_ordinal] + .0 + .send(NetworkClientMessages::Transaction { + transaction: SignedTransaction::send_money( + nonce, + from.clone(), + to.clone(), + &signer, + amount, + block_hash, + ), + is_forwarded: false, + check_only: false, + }) + .then(move |x| { + match x.unwrap() { + NetworkClientResponses::NoResponse | NetworkClientResponses::RequestRouted => { + assert_eq!(num_validators, 24); + send_tx( + num_validators, + connectors1, + (connector_ordinal + 8) % num_validators, + from, + to, + amount, + nonce, + block_hash, + ); } - future::ready(()) - }), - ); - } + NetworkClientResponses::ValidTx => { + println!("Transaction was received by validator {:?}", connector_ordinal); + } + other @ _ => { + println!( + "Transaction was rejected with an unexpected outcome: {:?}", + other + ); + assert!(false) + } + } + future::ready(()) + }), + ); +} - fn test_cross_shard_tx_callback( - res: Result, MailboxError>, - account_id: AccountId, - connectors: Arc, Addr)>>>, - iteration: Arc, - nonce: Arc, - validators: Vec, - successful_queries: Arc>>, - unsuccessful_queries: Arc, - balances: Arc>>, - observed_balances: Arc>>, - presumable_epoch: Arc>, - num_iters: usize, - block_hash: CryptoHash, - block_stats: Arc>, - min_ratio: Option, - max_ratio: Option, - ) { - let res = res.unwrap(); - - let query_response = match res { - Ok(query_response) => query_response, - Err(e) => { - println!("Query failed with {:?}", e); - *presumable_epoch.write().unwrap() += 1; - let connectors_ = connectors.write().unwrap(); - let connectors1 = connectors.clone(); - let iteration1 = iteration.clone(); - let nonce1 = nonce.clone(); - let validators1 = validators.clone(); - let successful_queries1 = successful_queries.clone(); - let unsuccessful_queries1 = unsuccessful_queries.clone(); - let balances1 = balances.clone(); - let observed_balances1 = observed_balances.clone(); - let presumable_epoch1 = presumable_epoch.clone(); - actix::spawn( - connectors_[account_id_to_shard_id(&account_id, 8) as usize - + (*presumable_epoch.read().unwrap() * 8) % 24] - .1 - .send(Query::new( - BlockReference::latest(), - QueryRequest::ViewAccount { account_id: account_id.clone() }, - )) - .then(move |x| { - test_cross_shard_tx_callback( - x, - account_id, - connectors1, - iteration1, - nonce1, - validators1, - successful_queries1, - unsuccessful_queries1, - balances1, - observed_balances1, - presumable_epoch1, - num_iters, - block_hash, - block_stats, - min_ratio, - max_ratio, - ); - future::ready(()) - }), - ); - return; +#[cfg(feature = "expensive_tests")] +fn test_cross_shard_tx_callback( + res: Result, MailboxError>, + account_id: AccountId, + connectors: Arc, Addr)>>>, + iteration: Arc, + nonce: Arc, + validators: Vec, + successful_queries: Arc>>, + unsuccessful_queries: Arc, + balances: Arc>>, + observed_balances: Arc>>, + presumable_epoch: Arc>, + num_iters: usize, + block_hash: CryptoHash, + block_stats: Arc>, + min_ratio: Option, + max_ratio: Option, +) { + let res = res.unwrap(); + + let query_response = match res { + Ok(query_response) => query_response, + Err(e) => { + println!("Query failed with {:?}", e); + *presumable_epoch.write().unwrap() += 1; + let connectors_ = connectors.write().unwrap(); + let connectors1 = connectors.clone(); + let iteration1 = iteration.clone(); + let nonce1 = nonce.clone(); + let validators1 = validators.clone(); + let successful_queries1 = successful_queries.clone(); + let unsuccessful_queries1 = unsuccessful_queries.clone(); + let balances1 = balances.clone(); + let observed_balances1 = observed_balances.clone(); + let presumable_epoch1 = presumable_epoch.clone(); + actix::spawn( + connectors_[account_id_to_shard_id(&account_id, 8) as usize + + (*presumable_epoch.read().unwrap() * 8) % 24] + .1 + .send(Query::new( + BlockReference::latest(), + QueryRequest::ViewAccount { account_id: account_id.clone() }, + )) + .then(move |x| { + test_cross_shard_tx_callback( + x, + account_id, + connectors1, + iteration1, + nonce1, + validators1, + successful_queries1, + unsuccessful_queries1, + balances1, + observed_balances1, + presumable_epoch1, + num_iters, + block_hash, + block_stats, + min_ratio, + max_ratio, + ); + future::ready(()) + }), + ); + return; + } + }; + + if let ViewAccount(view_account_result) = query_response.kind { + let mut expected = 0; + for i in 0..8 { + if validators[i] == account_id { + expected = balances.read().unwrap()[i]; + observed_balances.write().unwrap()[i] = view_account_result.amount; } - }; - - if let ViewAccount(view_account_result) = query_response.kind { - let mut expected = 0; - for i in 0..8 { - if validators[i] == account_id { - expected = balances.read().unwrap()[i]; - observed_balances.write().unwrap()[i] = view_account_result.amount; + } + + if view_account_result.amount == expected { + let mut successful_queries_local = successful_queries.write().unwrap(); + assert!(!successful_queries_local.contains(&account_id)); + successful_queries_local.insert(account_id.clone()); + if successful_queries_local.len() == 8 { + println!("Finished iteration {}", iteration.load(Ordering::Relaxed)); + + iteration.fetch_add(1, Ordering::Relaxed); + let iteration_local = iteration.load(Ordering::Relaxed); + if iteration_local > num_iters { + (&mut *block_stats.write().unwrap()).check_stats(true); + (&mut *block_stats.write().unwrap()).check_block_ratio(min_ratio, max_ratio); + System::current().stop(); } - } - if view_account_result.amount == expected { - let mut successful_queries_local = successful_queries.write().unwrap(); - assert!(!successful_queries_local.contains(&account_id)); - successful_queries_local.insert(account_id.clone()); - if successful_queries_local.len() == 8 { - println!("Finished iteration {}", iteration.load(Ordering::Relaxed)); - - iteration.fetch_add(1, Ordering::Relaxed); - let iteration_local = iteration.load(Ordering::Relaxed); - if iteration_local > num_iters { - (&mut *block_stats.write().unwrap()).check_stats(true); - (&mut *block_stats.write().unwrap()) - .check_block_ratio(min_ratio, max_ratio); - System::current().stop(); - } + let from = iteration_local % 8; + let to = (iteration_local / 8) % 8; + let amount = (5 + iteration_local) as u128; + let next_nonce = nonce.fetch_add(1, Ordering::Relaxed); + + send_tx( + validators.len(), + connectors.clone(), + account_id_to_shard_id(&validators[from], 8) as usize, + validators[from].clone(), + validators[to].clone(), + amount, + next_nonce as u64, + block_hash, + ); - let from = iteration_local % 8; - let to = (iteration_local / 8) % 8; - let amount = (5 + iteration_local) as u128; - let next_nonce = nonce.fetch_add(1, Ordering::Relaxed); - - send_tx( - validators.len(), - connectors.clone(), - account_id_to_shard_id(&validators[from], 8) as usize, - validators[from].clone(), - validators[to].clone(), - amount, - next_nonce as u64, - block_hash, - ); + let connectors_ = connectors.write().unwrap(); - let connectors_ = connectors.write().unwrap(); - - let mut balances_local = balances.write().unwrap(); - balances_local[from] -= amount; - balances_local[to] += amount; - - successful_queries_local.clear(); - unsuccessful_queries.store(0, Ordering::Relaxed); - - // Send the initial balance queries for the iteration - for i in 0..8 { - let connectors1 = connectors.clone(); - let iteration1 = iteration.clone(); - let nonce1 = nonce.clone(); - let validators1 = validators.clone(); - let successful_queries1 = successful_queries.clone(); - let unsuccessful_queries1 = unsuccessful_queries.clone(); - let balances1 = balances.clone(); - let observed_balances1 = observed_balances.clone(); - let presumable_epoch1 = presumable_epoch.clone(); - let account_id1 = validators[i].clone(); - let block_stats1 = block_stats.clone(); - actix::spawn( - connectors_[account_id_to_shard_id(&validators[i], 8) as usize - + (*presumable_epoch.read().unwrap() * 8) % 24] - .1 - .send(Query::new( - BlockReference::latest(), - QueryRequest::ViewAccount { account_id: validators[i].clone() }, - )) - .then(move |x| { - test_cross_shard_tx_callback( - x, - account_id1, - connectors1, - iteration1, - nonce1, - validators1, - successful_queries1, - unsuccessful_queries1, - balances1, - observed_balances1, - presumable_epoch1, - num_iters, - block_hash, - block_stats1, - min_ratio, - max_ratio, - ); - future::ready(()) - }), - ); - } + let mut balances_local = balances.write().unwrap(); + balances_local[from] -= amount; + balances_local[to] += amount; + + successful_queries_local.clear(); + unsuccessful_queries.store(0, Ordering::Relaxed); + + // Send the initial balance queries for the iteration + for i in 0..8 { + let connectors1 = connectors.clone(); + let iteration1 = iteration.clone(); + let nonce1 = nonce.clone(); + let validators1 = validators.clone(); + let successful_queries1 = successful_queries.clone(); + let unsuccessful_queries1 = unsuccessful_queries.clone(); + let balances1 = balances.clone(); + let observed_balances1 = observed_balances.clone(); + let presumable_epoch1 = presumable_epoch.clone(); + let account_id1 = validators[i].clone(); + let block_stats1 = block_stats.clone(); + actix::spawn( + connectors_[account_id_to_shard_id(&validators[i], 8) as usize + + (*presumable_epoch.read().unwrap() * 8) % 24] + .1 + .send(Query::new( + BlockReference::latest(), + QueryRequest::ViewAccount { account_id: validators[i].clone() }, + )) + .then(move |x| { + test_cross_shard_tx_callback( + x, + account_id1, + connectors1, + iteration1, + nonce1, + validators1, + successful_queries1, + unsuccessful_queries1, + balances1, + observed_balances1, + presumable_epoch1, + num_iters, + block_hash, + block_stats1, + min_ratio, + max_ratio, + ); + future::ready(()) + }), + ); } - } else { - // The balance is not correct, optionally trace, and resend the query - unsuccessful_queries.fetch_add(1, Ordering::Relaxed); - if unsuccessful_queries.load(Ordering::Relaxed) % 100 == 0 { - println!("Waiting for balances"); - print!("Expected: "); - for i in 0..8 { - print!("{} ", balances.read().unwrap()[i]); - } - println!(); - print!("Received: "); - for i in 0..8 { - print!("{} ", observed_balances.read().unwrap()[i]); - } - println!(); + } + } else { + // The balance is not correct, optionally trace, and resend the query + unsuccessful_queries.fetch_add(1, Ordering::Relaxed); + if unsuccessful_queries.load(Ordering::Relaxed) % 100 == 0 { + println!("Waiting for balances"); + print!("Expected: "); + for i in 0..8 { + print!("{} ", balances.read().unwrap()[i]); } - - let connectors_ = connectors.write().unwrap(); - let connectors1 = connectors.clone(); - let presumable_epoch1 = presumable_epoch.clone(); - actix::spawn( - connectors_[account_id_to_shard_id(&account_id, 8) as usize - + (*presumable_epoch.read().unwrap() * 8) % 24] - .1 - .send(Query::new( - BlockReference::latest(), - QueryRequest::ViewAccount { account_id: account_id.clone() }, - )) - .then(move |x| { - test_cross_shard_tx_callback( - x, - account_id, - connectors1, - iteration, - nonce, - validators, - successful_queries, - unsuccessful_queries, - balances, - observed_balances, - presumable_epoch1, - num_iters, - block_hash, - block_stats, - min_ratio, - max_ratio, - ); - future::ready(()) - }), - ); + println!(); + print!("Received: "); + for i in 0..8 { + print!("{} ", observed_balances.read().unwrap()[i]); + } + println!(); } + + let connectors_ = connectors.write().unwrap(); + let connectors1 = connectors.clone(); + let presumable_epoch1 = presumable_epoch.clone(); + actix::spawn( + connectors_[account_id_to_shard_id(&account_id, 8) as usize + + (*presumable_epoch.read().unwrap() * 8) % 24] + .1 + .send(Query::new( + BlockReference::latest(), + QueryRequest::ViewAccount { account_id: account_id.clone() }, + )) + .then(move |x| { + test_cross_shard_tx_callback( + x, + account_id, + connectors1, + iteration, + nonce, + validators, + successful_queries, + unsuccessful_queries, + balances, + observed_balances, + presumable_epoch1, + num_iters, + block_hash, + block_stats, + min_ratio, + max_ratio, + ); + future::ready(()) + }), + ); } } +} + +#[cfg(feature = "expensive_tests")] +fn test_cross_shard_tx_common( + num_iters: usize, + rotate_validators: bool, + drop_chunks: bool, + test_doomslug: bool, + block_production_time: u64, + min_ratio: Option, + max_ratio: Option, +) { + let validator_groups = 4; + init_integration_logger(); + run_actix(async move { + let connectors: Arc, Addr)>>> = + Arc::new(RwLock::new(vec![])); - fn test_cross_shard_tx_common( - num_iters: usize, - rotate_validators: bool, - drop_chunks: bool, - test_doomslug: bool, - block_production_time: u64, - min_ratio: Option, - max_ratio: Option, - ) { - let validator_groups = 4; - init_integration_logger(); - run_actix(async move { - let connectors: Arc, Addr)>>> = - Arc::new(RwLock::new(vec![])); - - let validators: Vec> = if rotate_validators { + let validators: Vec> = if rotate_validators { + [ [ - [ - "test1.1", "test1.2", "test1.3", "test1.4", "test1.5", "test1.6", - "test1.7", "test1.8", - ], - [ - "test2.1", "test2.2", "test2.3", "test2.4", "test2.5", "test2.6", - "test2.7", "test2.8", - ], - [ - "test3.1", "test3.2", "test3.3", "test3.4", "test3.5", "test3.6", - "test3.7", "test3.8", - ], - ] - .iter() - } else { - [[ "test1.1", "test1.2", "test1.3", "test1.4", "test1.5", "test1.6", "test1.7", "test1.8", - ]] - .iter() - } - .map(|l| l.iter().map(|account_id| account_id.parse().unwrap()).collect()) - .collect(); - let key_pairs = (0..32).map(|_| PeerInfo::random()).collect::>(); - let balances = Arc::new(RwLock::new(vec![])); - let observed_balances = Arc::new(RwLock::new(vec![])); - let presumable_epoch = Arc::new(RwLock::new(0usize)); - - let mut balances_local = balances.write().unwrap(); - let mut observed_balances_local = observed_balances.write().unwrap(); - for i in 0..8 { - balances_local.push(1000 + 100 * i); - observed_balances_local.push(0); - } + ], + [ + "test2.1", "test2.2", "test2.3", "test2.4", "test2.5", "test2.6", "test2.7", + "test2.8", + ], + [ + "test3.1", "test3.2", "test3.3", "test3.4", "test3.5", "test3.6", "test3.7", + "test3.8", + ], + ] + .iter() + } else { + [[ + "test1.1", "test1.2", "test1.3", "test1.4", "test1.5", "test1.6", "test1.7", + "test1.8", + ]] + .iter() + } + .map(|l| l.iter().map(|account_id| account_id.parse().unwrap()).collect()) + .collect(); + let key_pairs = (0..32).map(|_| PeerInfo::random()).collect::>(); + let balances = Arc::new(RwLock::new(vec![])); + let observed_balances = Arc::new(RwLock::new(vec![])); + let presumable_epoch = Arc::new(RwLock::new(0usize)); + + let mut balances_local = balances.write().unwrap(); + let mut observed_balances_local = observed_balances.write().unwrap(); + for i in 0..8 { + balances_local.push(1000 + 100 * i); + observed_balances_local.push(0); + } - let (genesis_block, conn, block_stats) = setup_mock_all_validators( - validators.clone(), - key_pairs.clone(), - validator_groups, - true, - block_production_time, - drop_chunks, - !test_doomslug, - 20, - test_doomslug, - vec![true; validators.iter().map(|x| x.len()).sum()], - vec![false; validators.iter().map(|x| x.len()).sum()], - true, - Arc::new(RwLock::new(Box::new( - move |_account_id: _, _msg: &PeerManagerMessageRequest| { - ( - PeerManagerMessageResponse::NetworkResponses( - NetworkResponses::NoResponse, - ), - true, - ) - }, - ))), - ); - *connectors.write().unwrap() = conn; - let block_hash = *genesis_block.hash(); + let (genesis_block, conn, block_stats) = setup_mock_all_validators( + validators.clone(), + key_pairs.clone(), + validator_groups, + true, + block_production_time, + drop_chunks, + !test_doomslug, + 20, + test_doomslug, + vec![true; validators.iter().map(|x| x.len()).sum()], + vec![false; validators.iter().map(|x| x.len()).sum()], + true, + Arc::new(RwLock::new(Box::new( + move |_account_id: _, _msg: &PeerManagerMessageRequest| { + ( + PeerManagerMessageResponse::NetworkResponses(NetworkResponses::NoResponse), + true, + ) + }, + ))), + ); + *connectors.write().unwrap() = conn; + let block_hash = *genesis_block.hash(); - let connectors_ = connectors.write().unwrap(); - let iteration = Arc::new(AtomicUsize::new(0)); - let nonce = Arc::new(AtomicUsize::new(1)); - let successful_queries = Arc::new(RwLock::new(HashSet::new())); - let unsuccessful_queries = Arc::new(AtomicUsize::new(0)); - let flat_validators = validators.iter().flatten().cloned().collect::>(); - - for i in 0..8 { - let connectors1 = connectors.clone(); - let iteration1 = iteration.clone(); - let nonce1 = nonce.clone(); - let flat_validators1 = flat_validators.clone(); - let successful_queries1 = successful_queries.clone(); - let unsuccessful_queries1 = unsuccessful_queries.clone(); - let balances1 = balances.clone(); - let observed_balances1 = observed_balances.clone(); - let presumable_epoch1 = presumable_epoch.clone(); - let account_id1 = flat_validators[i].clone(); - let block_stats1 = block_stats.clone(); - actix::spawn( - connectors_[account_id_to_shard_id(&flat_validators[i], 8) as usize - + *presumable_epoch.read().unwrap() * 8] - .1 - .send(Query::new( - BlockReference::latest(), - QueryRequest::ViewAccount { account_id: flat_validators[i].clone() }, - )) - .then(move |x| { - test_cross_shard_tx_callback( - x, - account_id1, - connectors1, - iteration1, - nonce1, - flat_validators1, - successful_queries1, - unsuccessful_queries1, - balances1, - observed_balances1, - presumable_epoch1, - num_iters, - block_hash, - block_stats1, - min_ratio, - max_ratio, - ); - future::ready(()) - }), - ); - } + let connectors_ = connectors.write().unwrap(); + let iteration = Arc::new(AtomicUsize::new(0)); + let nonce = Arc::new(AtomicUsize::new(1)); + let successful_queries = Arc::new(RwLock::new(HashSet::new())); + let unsuccessful_queries = Arc::new(AtomicUsize::new(0)); + let flat_validators = validators.iter().flatten().cloned().collect::>(); + + for i in 0..8 { + let connectors1 = connectors.clone(); + let iteration1 = iteration.clone(); + let nonce1 = nonce.clone(); + let flat_validators1 = flat_validators.clone(); + let successful_queries1 = successful_queries.clone(); + let unsuccessful_queries1 = unsuccessful_queries.clone(); + let balances1 = balances.clone(); + let observed_balances1 = observed_balances.clone(); + let presumable_epoch1 = presumable_epoch.clone(); + let account_id1 = flat_validators[i].clone(); + let block_stats1 = block_stats.clone(); + actix::spawn( + connectors_[account_id_to_shard_id(&flat_validators[i], 8) as usize + + *presumable_epoch.read().unwrap() * 8] + .1 + .send(Query::new( + BlockReference::latest(), + QueryRequest::ViewAccount { account_id: flat_validators[i].clone() }, + )) + .then(move |x| { + test_cross_shard_tx_callback( + x, + account_id1, + connectors1, + iteration1, + nonce1, + flat_validators1, + successful_queries1, + unsuccessful_queries1, + balances1, + observed_balances1, + presumable_epoch1, + num_iters, + block_hash, + block_stats1, + min_ratio, + max_ratio, + ); + future::ready(()) + }), + ); + } - near_network::test_utils::wait_or_panic(if rotate_validators { - 1000 * 60 * 80 - } else { - 1000 * 60 * 45 - }); + near_network::test_utils::wait_or_panic(if rotate_validators { + 1000 * 60 * 80 + } else { + 1000 * 60 * 45 }); - } + }); +} - #[test] - fn test_cross_shard_tx() { - test_cross_shard_tx_common(64, false, false, false, 70, Some(2.3), None); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx() { + test_cross_shard_tx_common(64, false, false, false, 70, Some(2.3), None); +} - #[test] - fn test_cross_shard_tx_doomslug() { - test_cross_shard_tx_common(64, false, false, true, 200, None, Some(1.5)); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx_doomslug() { + test_cross_shard_tx_common(64, false, false, true, 200, None, Some(1.5)); +} - #[test] - fn test_cross_shard_tx_drop_chunks() { - test_cross_shard_tx_common(64, false, true, false, 250, None, None); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx_drop_chunks() { + test_cross_shard_tx_common(64, false, true, false, 250, None, None); +} - #[test] - fn test_cross_shard_tx_8_iterations() { - test_cross_shard_tx_common(8, false, false, false, 200, Some(2.4), None); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx_8_iterations() { + test_cross_shard_tx_common(8, false, false, false, 200, Some(2.4), None); +} - #[test] - fn test_cross_shard_tx_8_iterations_drop_chunks() { - test_cross_shard_tx_common(8, false, true, false, 200, Some(2.4), None); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx_8_iterations_drop_chunks() { + test_cross_shard_tx_common(8, false, true, false, 200, Some(2.4), None); +} - #[test] - fn test_cross_shard_tx_with_validator_rotation_1() { - test_cross_shard_tx_common(8, true, false, false, 220, Some(2.4), None); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx_with_validator_rotation_1() { + test_cross_shard_tx_common(8, true, false, false, 220, Some(2.4), None); +} - #[test] - fn test_cross_shard_tx_with_validator_rotation_2() { - test_cross_shard_tx_common(24, true, false, false, 400, Some(2.4), None); - } +#[cfg(feature = "expensive_tests")] +#[test] +fn test_cross_shard_tx_with_validator_rotation_2() { + test_cross_shard_tx_common(24, true, false, false, 400, Some(2.4), None); } diff --git a/chain/client/src/tests/mod.rs b/chain/client/src/tests/mod.rs index 52390dba87f..f03f939d45d 100644 --- a/chain/client/src/tests/mod.rs +++ b/chain/client/src/tests/mod.rs @@ -1,6 +1,8 @@ mod bug_repros; +#[cfg(feature = "expensive_tests")] mod catching_up; mod chunks_management; +#[cfg(feature = "expensive_tests")] mod consensus; mod cross_shard_tx; mod query_client; diff --git a/nightly/expensive.txt b/nightly/expensive.txt index a256d2c2144..98df2f5f342 100644 --- a/nightly/expensive.txt +++ b/nightly/expensive.txt @@ -1,32 +1,32 @@ # catchup tests -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_third_epoch -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_third_epoch --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_last_block -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_last_block --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_distant_epoch -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_distant_epoch --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_skip_15 -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_skip_15 --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_send_15 -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_send_15 --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_non_zero_amounts -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_non_zero_amounts --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_height_6 -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_random_single_part_sync_height_6 --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_sanity_blocks_produced -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_sanity_blocks_produced --features nightly_protocol,nightly_protocol_features -expensive --timeout=3600 near-client near_client tests::catching_up::tests::test_all_chunks_accepted_1000 -expensive --timeout=3600 near-client near_client tests::catching_up::tests::test_all_chunks_accepted_1000 --features nightly_protocol,nightly_protocol_features -expensive --timeout=7200 near-client near_client tests::catching_up::tests::test_all_chunks_accepted_1000_slow -expensive --timeout=7200 near-client near_client tests::catching_up::tests::test_all_chunks_accepted_1000_slow --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_all_chunks_accepted_1000_rare_epoch_changing -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_all_chunks_accepted_1000_rare_epoch_changing --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_hold -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_catchup_receipts_sync_hold --features nightly_protocol,nightly_protocol_features -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_chunk_grieving -expensive --timeout=1800 near-client near_client tests::catching_up::tests::test_chunk_grieving --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_third_epoch +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_third_epoch --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_last_block +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_last_block --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_distant_epoch +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_distant_epoch --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_skip_15 +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_skip_15 --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_send_15 +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_send_15 --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_non_zero_amounts +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_non_zero_amounts --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_height_6 +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_height_6 --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_sanity_blocks_produced +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_sanity_blocks_produced --features nightly_protocol,nightly_protocol_features +expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 +expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 --features nightly_protocol,nightly_protocol_features +expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow +expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing +expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_hold +expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_hold --features nightly_protocol,nightly_protocol_features +expensive --timeout=1800 near-client near_client tests::catching_up::test_chunk_grieving +expensive --timeout=1800 near-client near_client tests::catching_up::test_chunk_grieving --features nightly_protocol,nightly_protocol_features expensive integration-tests test_catchup test_catchup expensive integration-tests test_catchup test_catchup --features nightly_protocol,nightly_protocol_features @@ -34,26 +34,26 @@ expensive integration-tests test_catchup test_catchup --features nightly_protoco # cross-shard transactions tests # TODO(#4618): Those tests are currently broken. Comment out while we’re # working on a fix / deciding whether to remove them. -# expensive --timeout=3000 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx -# expensive --timeout=3000 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx --features nightly_protocol,nightly_protocol_features -expensive --timeout=3000 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_doomslug -expensive --timeout=3000 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_doomslug --features nightly_protocol,nightly_protocol_features -expensive --timeout=3000 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_drop_chunks -expensive --timeout=3000 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_drop_chunks --features nightly_protocol,nightly_protocol_features +# expensive --timeout=3000 near-client near_client tests::cross_shard_tx::test_cross_shard_tx +# expensive --timeout=3000 near-client near_client tests::cross_shard_tx::test_cross_shard_tx --features nightly_protocol,nightly_protocol_features +expensive --timeout=3000 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_doomslug +expensive --timeout=3000 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_doomslug --features nightly_protocol,nightly_protocol_features +expensive --timeout=3000 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_drop_chunks +expensive --timeout=3000 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_drop_chunks --features nightly_protocol,nightly_protocol_features # TODO(#4618): Those tests are currently broken. Comment out while we’re # working on a fix / deciding whether to remove them. -# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_with_validator_rotation_1 -# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_with_validator_rotation_1 --features nightly_protocol,nightly_protocol_features -# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_with_validator_rotation_2 -# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_with_validator_rotation_2 --features nightly_protocol,nightly_protocol_features -# expensive --timeout=4800 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_8_iterations -# expensive --timeout=4800 near-client near_client tests::cross_shard_tx::tests::test_cross_shard_tx_8_iterations_drop_chunks +# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_with_validator_rotation_1 +# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_with_validator_rotation_1 --features nightly_protocol,nightly_protocol_features +# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_with_validator_rotation_2 +# expensive --timeout=5400 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_with_validator_rotation_2 --features nightly_protocol,nightly_protocol_features +# expensive --timeout=4800 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_8_iterations +# expensive --timeout=4800 near-client near_client tests::cross_shard_tx::test_cross_shard_tx_8_iterations_drop_chunks # consensus tests -expensive --timeout=3000 near-chain near_chain tests::doomslug::tests::test_fuzzy_doomslug_liveness_and_safety -expensive --timeout=3000 near-chain near_chain tests::doomslug::tests::test_fuzzy_doomslug_liveness_and_safety --features nightly_protocol,nightly_protocol_features -expensive --timeout=500 near-client near_client tests::consensus::tests::test_consensus_with_epoch_switches -expensive --timeout=500 near-client near_client tests::consensus::tests::test_consensus_with_epoch_switches --features nightly_protocol,nightly_protocol_features +expensive --timeout=3000 near-chain near_chain tests::doomslug::test_fuzzy_doomslug_liveness_and_safety +expensive --timeout=3000 near-chain near_chain tests::doomslug::test_fuzzy_doomslug_liveness_and_safety --features nightly_protocol,nightly_protocol_features +expensive --timeout=500 near-client near_client tests::consensus::test_consensus_with_epoch_switches +expensive --timeout=500 near-client near_client tests::consensus::test_consensus_with_epoch_switches --features nightly_protocol,nightly_protocol_features # testnet rpc expensive nearcore test_tps_regression test::test_highload @@ -117,18 +117,18 @@ expensive integration-tests standard_cases rpc::test::test_upload_contract_testn expensive integration-tests standard_cases rpc::test::test_upload_contract_testnet --features nightly_protocol,nightly_protocol_features # GC tests -expensive --timeout=900 near-chain near_chain tests::gc::tests::test_gc_remove_fork_large -expensive --timeout=900 near-chain near_chain tests::gc::tests::test_gc_remove_fork_large --features nightly_protocol,nightly_protocol_features -expensive --timeout=1200 near-chain near_chain tests::gc::tests::test_gc_not_remove_fork_large -expensive --timeout=1200 near-chain near_chain tests::gc::tests::test_gc_not_remove_fork_large --features nightly_protocol,nightly_protocol_features -expensive --timeout=1200 near-chain near_chain tests::gc::tests::test_gc_boundaries_large -expensive --timeout=1200 near-chain near_chain tests::gc::tests::test_gc_boundaries_large --features nightly_protocol,nightly_protocol_features -expensive --timeout=900 near-chain near_chain tests::gc::tests::test_gc_random_large -expensive --timeout=900 near-chain near_chain tests::gc::tests::test_gc_random_large --features nightly_protocol,nightly_protocol_features -expensive --timeout=600 near-chain near_chain tests::gc::tests::test_gc_pine -expensive --timeout=600 near-chain near_chain tests::gc::tests::test_gc_pine --features nightly_protocol,nightly_protocol_features -expensive --timeout=700 near-chain near_chain tests::gc::tests::test_gc_star_large -expensive --timeout=700 near-chain near_chain tests::gc::tests::test_gc_star_large --features nightly_protocol,nightly_protocol_features +expensive --timeout=900 near-chain near_chain tests::gc::test_gc_remove_fork_large +expensive --timeout=900 near-chain near_chain tests::gc::test_gc_remove_fork_large --features nightly_protocol,nightly_protocol_features +expensive --timeout=1200 near-chain near_chain tests::gc::test_gc_not_remove_fork_large +expensive --timeout=1200 near-chain near_chain tests::gc::test_gc_not_remove_fork_large --features nightly_protocol,nightly_protocol_features +expensive --timeout=1200 near-chain near_chain tests::gc::test_gc_boundaries_large +expensive --timeout=1200 near-chain near_chain tests::gc::test_gc_boundaries_large --features nightly_protocol,nightly_protocol_features +expensive --timeout=900 near-chain near_chain tests::gc::test_gc_random_large +expensive --timeout=900 near-chain near_chain tests::gc::test_gc_random_large --features nightly_protocol,nightly_protocol_features +expensive --timeout=600 near-chain near_chain tests::gc::test_gc_pine +expensive --timeout=600 near-chain near_chain tests::gc::test_gc_pine --features nightly_protocol,nightly_protocol_features +expensive --timeout=700 near-chain near_chain tests::gc::test_gc_star_large +expensive --timeout=700 near-chain near_chain tests::gc::test_gc_star_large --features nightly_protocol,nightly_protocol_features expensive integration-tests client process_blocks::test_gc_after_state_sync expensive integration-tests client process_blocks::test_gc_after_state_sync --features nightly_protocol,nightly_protocol_features