feat(sandbox): fast forwarding (near#6158)

* Added fast-forwarding to sandbox

* Changed name to delta_height for better clarity

* Remove print statements

* Cargo format

* Added sandbox fast forward test

* Move import into inplaced location as full path

* Move other sandbox imports into inplaced locations

* Added python test for sandbox fast forwarding

* Format python test

* Fix import issue from updating to latest

* Addressed comments

* Removed unnecessary conditional flags for sandbox

* Use utils module over direct import for patch_state

chain/chain/src/chain.rs

@@ -3947,11 +3947,14 @@ impl<'a> ChainUpdate<'a> {
         let head = self.chain_store_update.head()?;
         let is_next = block.header().prev_hash() == &head.last_block_hash;
 
-        // A heuristic to prevent block height to jump too fast towards BlockHeight::max and cause
-        // overflow-related problems
-        let block_height = block.header().height();
-        if block_height > head.height + self.epoch_length * 20 {
-            return Err(ErrorKind::InvalidBlockHeight(block_height).into());
+        // Sandbox allows fast-forwarding, so only enable when not within sandbox
+        if !cfg!(feature = "sandbox") {
+            // A heuristic to prevent block height to jump too fast towards BlockHeight::max and cause
+            // overflow-related problems
+            let block_height = block.header().height();
+            if block_height > head.height + self.epoch_length * 20 {
+                return Err(ErrorKind::InvalidBlockHeight(block_height).into());
+            }
         }
 
         // Block is an orphan if we do not know about the previous full block.

chain/client/src/client.rs

@@ -944,6 +944,22 @@ impl Client {
         Ok(())
     }
 
+    #[cfg(feature = "sandbox")]
+    pub fn sandbox_update_tip(&mut self, height: BlockHeight) -> Result<(), Error> {
+        let tip = self.chain.head()?;
+
+        let last_final_hash =
+            *self.chain.get_block_header(&tip.last_block_hash)?.last_final_block();
+        let last_final_height = if last_final_hash == CryptoHash::default() {
+            self.chain.genesis().height()
+        } else {
+            self.chain.get_block_header(&last_final_hash)?.height()
+        };
+        self.doomslug.set_tip(Clock::instant(), tip.last_block_hash, height, last_final_height);
+
+        Ok(())
+    }
+
     pub fn send_approval(
         &mut self,
         parent_hash: &CryptoHash,

chain/client/src/client_actor.rs

@@ -88,6 +88,9 @@ pub struct ClientActor {
     block_catch_up_scheduler: Box<dyn Fn(BlockCatchUpRequest)>,
     state_split_scheduler: Box<dyn Fn(StateSplitRequest)>,
     state_parts_client_arbiter: Arbiter,
+
+    #[cfg(feature = "sandbox")]
+    fastforward_delta: Option<near_primitives::types::BlockHeightDelta>,
 }
 
 /// Blocks the program until given genesis time arrives.
@@ -182,6 +185,9 @@ impl ClientActor {
                 sync_jobs_actor_addr,
             ),
             state_parts_client_arbiter: state_parts_arbiter,
+
+            #[cfg(feature = "sandbox")]
+            fastforward_delta: None,
         })
     }
 }
@@ -352,6 +358,10 @@ impl Handler<NetworkClientMessages> for ClientActor {
                             ),
                         )
                     }
+                    near_network_primitives::types::NetworkSandboxMessage::SandboxFastForward(delta_height) => {
+                        self.fastforward_delta = Some(delta_height);
+                        NetworkClientResponses::NoResponse
+                    }
                 };
             }
             NetworkClientMessages::Transaction { transaction, is_forwarded, check_only } => {
@@ -764,6 +774,21 @@ impl ClientActor {
 
         let head = self.client.chain.head()?;
         let latest_known = self.client.chain.mut_store().get_latest_known()?;
+
+        #[cfg(feature = "sandbox")]
+        let latest_known = if let Some(delta_height) = self.fastforward_delta.take() {
+            let new_latest_known = near_chain::types::LatestKnown {
+                height: latest_known.height + delta_height,
+                seen: near_primitives::utils::to_timestamp(Clock::utc()),
+            };
+
+            self.client.chain.mut_store().save_latest_known(new_latest_known.clone())?;
+            self.client.sandbox_update_tip(new_latest_known.height)?;
+            new_latest_known
+        } else {
+            latest_known
+        };
+
         assert!(
             head.height <= latest_known.height,
             "Latest known height is invalid {} vs {}",

chain/jsonrpc-primitives/src/types/sandbox.rs

@@ -1,4 +1,5 @@
 use near_primitives::state_record::StateRecord;
+use near_primitives::types::BlockHeightDelta;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
@@ -43,3 +44,45 @@ impl From<RpcSandboxPatchStateError> for crate::errors::RpcError {
         Self::new_internal_or_handler_error(Some(error_data.clone()), error_data)
     }
 }
+
+#[derive(Deserialize, Serialize)]
+pub struct RpcSandboxFastForwardRequest {
+    pub delta_height: BlockHeightDelta,
+}
+
+impl RpcSandboxFastForwardRequest {
+    pub fn parse(value: Option<Value>) -> Result<Self, crate::errors::RpcParseError> {
+        Ok(crate::utils::parse_params::<RpcSandboxFastForwardRequest>(value)?)
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct RpcSandboxFastForwardResponse {}
+
+#[derive(thiserror::Error, Debug, Serialize, Deserialize)]
+#[serde(tag = "name", content = "info", rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum RpcSandboxFastForwardError {
+    #[error("The node reached its limits. Try again later. More details: {error_message}")]
+    InternalError { error_message: String },
+}
+
+impl From<actix::MailboxError> for RpcSandboxFastForwardError {
+    fn from(error: actix::MailboxError) -> Self {
+        Self::InternalError { error_message: error.to_string() }
+    }
+}
+
+impl From<RpcSandboxFastForwardError> for crate::errors::RpcError {
+    fn from(error: RpcSandboxFastForwardError) -> Self {
+        let error_data = match serde_json::to_value(error) {
+            Ok(value) => value,
+            Err(err) => {
+                return Self::new_internal_error(
+                    None,
+                    format!("Failed to serialize RpcSandboxFastForwardError: {:?}", err),
+                )
+            }
+        };
+        Self::new_internal_or_handler_error(Some(error_data.clone()), error_data)
+    }
+}

chain/jsonrpc/src/lib.rs

@@ -560,6 +560,17 @@ impl JsonRpcHandler {
                 serde_json::to_value(sandbox_patch_state_response)
                     .map_err(|err| RpcError::serialization_error(err.to_string()))
             }
+            #[cfg(feature = "sandbox")]
+            "sandbox_fast_forward" => {
+                let sandbox_fast_forward_request =
+                    near_jsonrpc_primitives::types::sandbox::RpcSandboxFastForwardRequest::parse(
+                        request.params,
+                    )?;
+                let sandbox_fast_forward_response =
+                    self.sandbox_fast_forward(sandbox_fast_forward_request).await?;
+                serde_json::to_value(sandbox_fast_forward_response)
+                    .map_err(|err| RpcError::serialization_error(err.to_string()))
+            }
             _ => Err(RpcError::method_not_found(request.method.clone())),
         };
 
@@ -1133,6 +1144,23 @@ impl JsonRpcHandler {
 
         Ok(near_jsonrpc_primitives::types::sandbox::RpcSandboxPatchStateResponse {})
     }
+
+    async fn sandbox_fast_forward(
+        &self,
+        fast_forward_request: near_jsonrpc_primitives::types::sandbox::RpcSandboxFastForwardRequest,
+    ) -> Result<
+        near_jsonrpc_primitives::types::sandbox::RpcSandboxFastForwardResponse,
+        near_jsonrpc_primitives::types::sandbox::RpcSandboxFastForwardError,
+    > {
+        self.client_addr
+            .send(NetworkClientMessages::Sandbox(
+                near_network_primitives::types::NetworkSandboxMessage::SandboxFastForward(
+                    fast_forward_request.delta_height,
+                ),
+            ))
+            .await?;
+        Ok(near_jsonrpc_primitives::types::sandbox::RpcSandboxFastForwardResponse {})
+    }
 }
 
 #[cfg(feature = "test_features")]

chain/network-primitives/src/types.rs

@@ -276,6 +276,7 @@ pub enum NetworkAdversarialMessage {
 pub enum NetworkSandboxMessage {
     SandboxPatchState(Vec<near_primitives::state_record::StateRecord>),
     SandboxPatchStateStatus,
+    SandboxFastForward(near_primitives::types::BlockHeightDelta),
 }
 
 #[cfg(feature = "sandbox")]

integration-tests/src/tests/client/sandbox.rs

@@ -1,10 +1,19 @@
 use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
+use actix::System;
+
+use near_actix_test_utils::run_actix;
 use near_chain::{ChainGenesis, Provenance, RuntimeAdapter};
 use near_chain_configs::Genesis;
-use near_client::test_utils::TestEnv;
+use near_client::test_utils::{setup_mock, TestEnv};
 use near_crypto::{InMemorySigner, KeyType};
+use near_logger_utils::init_test_logger;
+use near_network::types::{
+    NetworkClientMessages, NetworkRequests, NetworkResponses, PeerManagerMessageResponse,
+};
+use near_network_primitives::types::NetworkSandboxMessage;
 use near_primitives::account::Account;
 use near_primitives::serialize::{from_base64, to_base64};
 use near_primitives::state_record::StateRecord;
@@ -77,7 +86,6 @@ fn send_tx(
 }
 
 #[test]
-#[cfg(feature = "sandbox")]
 fn test_patch_state() {
     let (mut env, _signer) = test_setup();
 
@@ -95,7 +103,6 @@ fn test_patch_state() {
 }
 
 #[test]
-#[cfg(feature = "sandbox")]
 fn test_patch_account() {
     let (mut env, _signer) = test_setup();
     let mut test1: Account = env.query_account("test1".parse().unwrap()).into();
@@ -109,3 +116,39 @@ fn test_patch_account() {
     let test1_after = env.query_account("test1".parse().unwrap());
     assert_eq!(test1_after.amount, 10);
 }
+
+#[test]
+fn test_fast_forward() {
+    init_test_logger();
+    run_actix(async {
+        let count = Arc::new(AtomicUsize::new(0));
+        // Produce 20 blocks
+        let (client, _view_client) = setup_mock(
+            vec!["test".parse().unwrap()],
+            "test".parse().unwrap(),
+            true,
+            false,
+            Box::new(move |msg, _ctx, _| {
+                if let NetworkRequests::Block { block } = msg.as_network_requests_ref() {
+                    let height = block.header().height();
+                    count.fetch_add(1, Ordering::Relaxed);
+                    if count.load(Ordering::Relaxed) >= 20 {
+                        assert!(
+                            height >= 10000,
+                            "Was not able to fast forward. Current height: {}",
+                            height
+                        );
+                        System::current().stop();
+                    }
+                }
+                PeerManagerMessageResponse::NetworkResponses(NetworkResponses::NoResponse)
+            }),
+        );
+
+        // Fast forward by 10,000 blocks:
+        client.do_send(NetworkClientMessages::Sandbox(NetworkSandboxMessage::SandboxFastForward(
+            10000,
+        )));
+        near_network::test_utils::wait_or_panic(5000);
+    });
+}

nightly/sandbox.txt

@@ -1,2 +1,3 @@
 # python sandbox node tests
 pytest sandbox/patch_state.py --features sandbox
+pytest sandbox/fast_forward.py --features sandbox

pytest/lib/utils.py

@@ -408,3 +408,26 @@ def wait_for_blocks(node: cluster.LocalNode,
         logger.info(f'{latest}  (waiting for #{target})')
         if latest.height >= target:
             return latest
+
+
+def figure_out_sandbox_binary():
+    config = {
+        'local': True,
+        'release': False,
+    }
+    repo_dir = pathlib.Path(__file__).resolve().parents[2]
+    # When run on NayDuck we end up with a binary called neard in target/debug
+    # but when run locally the binary might be neard-sandbox or near-sandbox
+    # instead.  Try to figure out whichever binary is available and use that.
+    for release in ('release', 'debug'):
+        root = repo_dir / 'target' / release
+        for exe in ('neard-sandbox', 'near-sandbox', 'neard'):
+            if (root / exe).exists():
+                logger.info(
+                    f'Using {(root / exe).relative_to(repo_dir)} binary')
+                config['near_root'] = str(root)
+                config['binary_name'] = exe
+                return config
+
+    assert False, ('Unable to figure out location of neard-sandbox binary; '
+                   'Did you forget to run `make sandbox`?')

pytest/tests/sandbox/fast_forward.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# test fast fowarding by a specific block height within a sandbox node. This will
+# fail if the block height is not past the forwarded height.
+
+import sys, time
+import pathlib
+
+sys.path.append(str(pathlib.Path(__file__).resolve().parents[2] / 'lib'))
+
+from cluster import start_cluster
+from utils import figure_out_sandbox_binary
+
+# startup a RPC node
+BLOCKS_TO_FASTFORARD = 10000
+CONFIG = figure_out_sandbox_binary()
+nodes = start_cluster(1, 0, 1, CONFIG, [["epoch_length", 10]], {})
+
+# request to fast forward
+nodes[0].json_rpc('sandbox_fast_forward', {
+    "delta_height": BLOCKS_TO_FASTFORARD,
+})
+
+# wait a little for it to fast forward
+time.sleep(3)
+
+# Assert at the end that the node is past the amounts of blocks we specified
+assert nodes[0].get_latest_block().height > BLOCKS_TO_FASTFORARD

pytest/tests/sandbox/patch_state.py

@@ -2,50 +2,24 @@
 # Patch contract states in a sandbox node
 
 import sys, time
-import base58
 import base64
 import pathlib
 
 sys.path.append(str(pathlib.Path(__file__).resolve().parents[2] / 'lib'))
 
+import utils
 from cluster import start_cluster
-from configured_logger import logger
 from transaction import sign_deploy_contract_tx, sign_function_call_tx
-from utils import load_test_contract
 
-CONFIG = {
-    'local': True,
-    'release': False,
-}
-
-
-def figure_out_binary():
-    repo_dir = pathlib.Path(__file__).resolve().parents[3]
-    # When run on NayDuck we end up with a binary called neard in target/debug
-    # but when run locally the binary might be neard-sandbox or near-sandbox
-    # instead.  Try to figure out whichever binary is available and use that.
-    for release in ('release', 'debug'):
-        root = repo_dir / 'target' / release
-        for exe in ('neard-sandbox', 'near-sandbox', 'neard'):
-            if (root / exe).exists():
-                logger.info(
-                    f'Using {(root / exe).relative_to(repo_dir)} binary')
-                CONFIG['near_root'] = str(root)
-                CONFIG['binary_name'] = exe
-                return
-    assert False, ('Unable to figure out location of neard-sandbox binary; '
-                   'Did you forget to run `make sandbox`?')
-
-
-figure_out_binary()
+CONFIG = utils.figure_out_sandbox_binary()
 
 # start node
 nodes = start_cluster(1, 0, 1, CONFIG, [["epoch_length", 10]], {})
 
 # deploy contract
 hash_ = nodes[0].get_latest_block().hash_bytes
-tx = sign_deploy_contract_tx(nodes[0].signer_key, load_test_contract(), 10,
-                             hash_)
+tx = sign_deploy_contract_tx(nodes[0].signer_key, utils.load_test_contract(),
+                             10, hash_)
 nodes[0].send_tx(tx)
 time.sleep(3)