Skip to content

Commit

Permalink
[Debug UI] Add cold storage support to Entity Debug. (near#12086)
Browse files Browse the repository at this point in the history 
Entity Debug UI now shows a little "Use cold storage" checkbox at the
bottom. Any new query that is being sent will use the current state of
the checkbox to decide whether it should query from cold storage.

Also run "npm run fix" to fix file formatting of all the tsx files.
  • Loading branch information
@robin-near
robin-near authored Sep 12, 2024
1 parent f8440da commit 328a8da
Show file tree
Hide file tree
Showing 22 changed files with 247 additions and 170 deletions.
12 changes: 10 additions & 2 deletions chain/jsonrpc-primitives/src/types/entity_debug.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,25 @@ pub enum EntityQuery {
ValidatorAssignmentsAtHeight { block_height: BlockHeight, epoch_id: EpochId },
}

#[derive(Serialize, Deserialize)]
pub struct EntityQueryWithParams {
#[serde(flatten)]
pub query: EntityQuery,
#[serde(default)]
pub use_cold_storage: bool,
}

/// We use a trait for this, because jsonrpc does not have access to low-level
/// blockchain data structures for implementing the queries.
pub trait EntityDebugHandler: Sync + Send {
fn query(&self, query: EntityQuery) -> Result<EntityDataValue, RpcError>;
fn query(&self, query: EntityQueryWithParams) -> Result<EntityDataValue, RpcError>;
}

/// For tests.
pub struct DummyEntityDebugHandler {}

impl EntityDebugHandler for DummyEntityDebugHandler {
fn query(&self, _query: EntityQuery) -> Result<EntityDataValue, RpcError> {
fn query(&self, _query: EntityQueryWithParams) -> Result<EntityDataValue, RpcError> {
Err(RpcError::new_internal_error(None, "Not implemented".to_string()))
}
}
4 changes: 2 additions & 2 deletions chain/jsonrpc/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ pub use near_jsonrpc_client as client;
use near_jsonrpc_primitives::errors::{RpcError, RpcErrorKind};
use near_jsonrpc_primitives::message::{Message, Request};
use near_jsonrpc_primitives::types::config::{RpcProtocolConfigError, RpcProtocolConfigResponse};
use near_jsonrpc_primitives::types::entity_debug::{EntityDebugHandler, EntityQuery};
use near_jsonrpc_primitives::types::entity_debug::{EntityDebugHandler, EntityQueryWithParams};
use near_jsonrpc_primitives::types::query::RpcQueryRequest;
use near_jsonrpc_primitives::types::split_storage::{
RpcSplitStorageInfoRequest, RpcSplitStorageInfoResponse,
Expand Down Expand Up @@ -1407,7 +1407,7 @@ async fn debug_handler(
}

async fn handle_entity_debug(
req: web::Json<EntityQuery>,
req: web::Json<EntityQueryWithParams>,
handler: web::Data<JsonRpcHandler>,
) -> Result<HttpResponse, HttpError> {
match handler.entity_debug_handler.query(req.0) {
Expand Down
107 changes: 46 additions & 61 deletions nearcore/src/entity_debug.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use near_epoch_manager::EpochManagerAdapter;
use near_jsonrpc_primitives::errors::RpcError;
use near_jsonrpc_primitives::types::entity_debug::{
EntityDataEntry, EntityDataStruct, EntityDataValue, EntityDebugHandler, EntityQuery,
EntityQueryWithParams,
};
use near_primitives::block::Tip;
use near_primitives::challenge::{PartialState, TrieValue};
Expand Down Expand Up @@ -47,19 +48,19 @@ use std::sync::Arc;
pub struct EntityDebugHandlerImpl {
pub epoch_manager: Arc<dyn EpochManagerAdapter>,
pub runtime: Arc<dyn RuntimeAdapter>,
pub store: Store,
pub hot_store: Store,
pub cold_store: Option<Store>,
}

impl EntityDebugHandlerImpl {
fn query_impl(&self, query: EntityQuery) -> anyhow::Result<EntityDataValue> {
fn query_impl(&self, store: Store, query: EntityQuery) -> anyhow::Result<EntityDataValue> {
match query {
EntityQuery::AllShardsByEpochId { epoch_id } => {
let shard_layout = self.epoch_manager.get_shard_layout(&epoch_id)?;
Ok(serialize_entity(&shard_layout.shard_uids().collect::<Vec<_>>()))
}
EntityQuery::BlockByHash { block_hash } => {
let block = self
.store
let block = store
.get_ser::<Block>(DBCol::Block, &borsh::to_vec(&block_hash).unwrap())?
.ok_or_else(|| anyhow!("Block not found"))?;
let author = self
Expand All @@ -76,8 +77,7 @@ impl EntityDebugHandlerImpl {
Ok(ret)
}
EntityQuery::BlockHashByHeight { block_height } => {
let block_hash = self
.store
let block_hash = store
.get_ser::<CryptoHash>(
DBCol::BlockHeight,
&borsh::to_vec(&block_height).unwrap(),
Expand All @@ -86,8 +86,7 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&block_hash))
}
EntityQuery::BlockHeaderByHash { block_hash } => {
let block_header = self
.store
let block_header = store
.get_ser::<BlockHeader>(
DBCol::BlockHeader,
&borsh::to_vec(&block_hash).unwrap(),
Expand All @@ -96,15 +95,13 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&BlockHeaderView::from(block_header)))
}
EntityQuery::BlockInfoByHash { block_hash } => {
let block_info = self
.store
let block_info = store
.get_ser::<BlockInfo>(DBCol::BlockInfo, &borsh::to_vec(&block_hash).unwrap())?
.ok_or_else(|| anyhow!("BlockInfo not found"))?;
Ok(serialize_entity(&block_info))
}
EntityQuery::BlockMerkleTreeByHash { block_hash } => {
let block_merkle_tree = self
.store
let block_merkle_tree = store
.get_ser::<PartialMerkleTree>(
DBCol::BlockMerkleTree,
&borsh::to_vec(&block_hash).unwrap(),
Expand All @@ -113,12 +110,11 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&block_merkle_tree))
}
EntityQuery::BlockMisc(()) => {
let block_misc = BlockMiscData::from_store(&self.store)?;
let block_misc = BlockMiscData::from_store(&store)?;
Ok(serialize_entity(&block_misc))
}
EntityQuery::ChunkByHash { chunk_hash } => {
let chunk = self
.store
let chunk = store
.get_ser::<ShardChunk>(DBCol::Chunks, &borsh::to_vec(&chunk_hash).unwrap())?
.ok_or_else(|| anyhow!("Chunk not found"))?;
let epoch_id =
Expand All @@ -131,8 +127,7 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&ChunkView::from_author_chunk(author, chunk)))
}
EntityQuery::ChunkExtraByBlockHashShardUId { block_hash, shard_uid } => {
let chunk_extra = self
.store
let chunk_extra = store
.get_ser::<ChunkExtra>(
DBCol::ChunkExtra,
&get_block_shard_uid(&block_hash, &shard_uid),
Expand All @@ -141,12 +136,10 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&chunk_extra))
}
EntityQuery::ChunkExtraByChunkHash { chunk_hash } => {
let chunk = self
.store
let chunk = store
.get_ser::<ShardChunk>(DBCol::Chunks, &borsh::to_vec(&chunk_hash).unwrap())?
.ok_or_else(|| anyhow!("Chunk not found"))?;
let block_hash = self
.store
let block_hash = store
.get_ser::<CryptoHash>(
DBCol::BlockHeight,
&borsh::to_vec(&chunk.height_included()).unwrap(),
Expand All @@ -156,8 +149,7 @@ impl EntityDebugHandlerImpl {
self.epoch_manager.get_epoch_id_from_prev_block(chunk.prev_block())?;
let shard_id = chunk.shard_id();
let shard_uid = self.epoch_manager.shard_id_to_uid(shard_id, &epoch_id)?;
let chunk_extra = self
.store
let chunk_extra = store
.get_ser::<ChunkExtra>(
DBCol::ChunkExtra,
&get_block_shard_uid(&block_hash, &shard_uid),
Expand All @@ -166,8 +158,7 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&chunk_extra))
}
EntityQuery::EpochInfoAggregator(()) => {
let aggregator = self
.store
let aggregator = store
.get_ser::<EpochInfoAggregator>(DBCol::EpochInfo, AGGREGATOR_KEY)?
.ok_or_else(|| anyhow!("Aggregator not found"))?;
Ok(serialize_entity(&aggregator))
Expand All @@ -177,19 +168,17 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&*epoch_info))
}
EntityQuery::FlatStateByTrieKey { trie_key, shard_uid } => {
let state = self
.store
let state = store
.get_ser::<FlatStateValue>(
DBCol::FlatState,
&encode_flat_state_db_key(shard_uid, &hex::decode(&trie_key)?),
)?
.ok_or_else(|| anyhow!("Flat state not found"))?;
let data = self.deref_flat_state_value(state, shard_uid)?;
let data = self.deref_flat_state_value(&store, state, shard_uid)?;
Ok(serialize_entity(&hex::encode(&data)))
}
EntityQuery::FlatStateChangesByBlockHash { block_hash, shard_uid } => {
let changes = self
.store
let changes = store
.get_ser::<FlatStateChanges>(
DBCol::FlatStateChanges,
&borsh::to_vec(&KeyForFlatStateDelta { block_hash, shard_uid }).unwrap(),
Expand All @@ -199,16 +188,17 @@ impl EntityDebugHandlerImpl {
for (key, value) in changes.0.into_iter() {
let key = hex::encode(&key);
let value = match value {
Some(v) => Some(hex::encode(&self.deref_flat_state_value(v, shard_uid)?)),
Some(v) => {
Some(hex::encode(&self.deref_flat_state_value(&store, v, shard_uid)?))
}
None => None,
};
changes_view.push(FlatStateChangeView { key, value });
}
Ok(serialize_entity(&changes_view))
}
EntityQuery::FlatStateDeltaMetadataByBlockHash { block_hash, shard_uid } => {
let metadata = self
.store
let metadata = store
.get_ser::<FlatStateDeltaMetadata>(
DBCol::FlatStateDeltaMetadata,
&borsh::to_vec(&KeyForFlatStateDelta { block_hash, shard_uid }).unwrap(),
Expand All @@ -217,8 +207,7 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&metadata))
}
EntityQuery::FlatStorageStatusByShardUId { shard_uid } => {
let status = self
.store
let status = store
.get_ser::<FlatStorageStatus>(
DBCol::FlatStorageStatus,
&borsh::to_vec(&shard_uid).unwrap(),
Expand All @@ -227,8 +216,7 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&status))
}
EntityQuery::NextBlockHashByHash { block_hash } => {
let next_block_hash = self
.store
let next_block_hash = store
.get_ser::<CryptoHash>(
DBCol::NextBlockHashes,
&borsh::to_vec(&block_hash).unwrap(),
Expand All @@ -238,8 +226,7 @@ impl EntityDebugHandlerImpl {
}
EntityQuery::OutcomeByTransactionHash { transaction_hash: outcome_id }
| EntityQuery::OutcomeByReceiptId { receipt_id: outcome_id } => {
let (_, outcome) = self
.store
let (_, outcome) = store
.iter_prefix_ser::<ExecutionOutcomeWithProof>(
DBCol::TransactionResultForBlock,
&borsh::to_vec(&outcome_id).unwrap(),
Expand All @@ -253,8 +240,7 @@ impl EntityDebugHandlerImpl {
block_hash,
}
| EntityQuery::OutcomeByReceiptIdAndBlockHash { receipt_id: outcome_id, block_hash } => {
let outcome = self
.store
let outcome = store
.get_ser::<ExecutionOutcomeWithProof>(
DBCol::TransactionResultForBlock,
&get_outcome_id_block_hash(&outcome_id, &block_hash),
Expand All @@ -263,8 +249,7 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&ExecutionOutcomeView::from(outcome.outcome)))
}
EntityQuery::ReceiptById { receipt_id } => {
let receipt = self
.store
let receipt = store
.get_ser::<Receipt>(DBCol::Receipts, &borsh::to_vec(&receipt_id).unwrap())?
.ok_or_else(|| anyhow!("Receipt not found"))?;
Ok(serialize_entity(&ReceiptView::from(receipt)))
Expand All @@ -287,17 +272,15 @@ impl EntityDebugHandlerImpl {
Ok(serialize_entity(&shard_uid))
}
EntityQuery::StateTransitionData { block_hash } => {
let block = self
.store
let block = store
.get_ser::<Block>(DBCol::Block, &borsh::to_vec(&block_hash).unwrap())?
.ok_or_else(|| anyhow!("Block not found"))?;
let epoch_id = block.header().epoch_id();
let shard_layout = self.epoch_manager.get_shard_layout(&epoch_id)?;
let shard_ids = shard_layout.shard_ids().collect::<Vec<_>>();
let mut state_transitions = EntityDataStruct::new();
for shard_id in shard_ids {
let state_transition = self
.store
let state_transition = store
.get_ser::<StoredChunkStateTransitionData>(
DBCol::StateTransitionData,
&get_block_shard_id(&block_hash, shard_id),
Expand All @@ -318,29 +301,25 @@ impl EntityDebugHandlerImpl {
Ok(EntityDataValue::Struct(state_transitions.into()))
}
EntityQuery::TipAtFinalHead(_) => {
let tip = self
.store
let tip = store
.get_ser::<Tip>(DBCol::BlockMisc, FINAL_HEAD_KEY)?
.ok_or_else(|| anyhow!("Tip not found"))?;
Ok(serialize_entity(&tip))
}
EntityQuery::TipAtHead(_) => {
let tip = self
.store
let tip = store
.get_ser::<Tip>(DBCol::BlockMisc, HEAD_KEY)?
.ok_or_else(|| anyhow!("Tip not found"))?;
Ok(serialize_entity(&tip))
}
EntityQuery::TipAtHeaderHead(_) => {
let tip = self
.store
let tip = store
.get_ser::<Tip>(DBCol::BlockMisc, HEADER_HEAD_KEY)?
.ok_or_else(|| anyhow!("Tip not found"))?;
Ok(serialize_entity(&tip))
}
EntityQuery::TransactionByHash { transaction_hash } => {
let transaction = self
.store
let transaction = store
.get_ser::<SignedTransaction>(
DBCol::Transactions,
&borsh::to_vec(&transaction_hash).unwrap(),
Expand Down Expand Up @@ -445,8 +424,7 @@ impl EntityDebugHandlerImpl {
Ok(EntityDataValue::Struct(Box::new(entity_data)))
}
EntityQuery::TrieRootByChunkHash { chunk_hash } => {
let chunk = self
.store
let chunk = store
.get_ser::<ShardChunk>(DBCol::Chunks, &borsh::to_vec(&chunk_hash).unwrap())?
.ok_or_else(|| anyhow!("Chunk not found"))?;
let shard_layout = self
Expand Down Expand Up @@ -512,12 +490,12 @@ impl EntityDebugHandlerImpl {

fn deref_flat_state_value(
&self,
store: &Store,
state: FlatStateValue,
shard_uid: ShardUId,
) -> anyhow::Result<Vec<u8>> {
Ok(match state {
FlatStateValue::Ref(value) => self
.store
FlatStateValue::Ref(value) => store
.get(
DBCol::State,
&TrieCachingStorage::get_key_from_shard_uid_and_hash(shard_uid, &value.hash),
Expand All @@ -530,8 +508,15 @@ impl EntityDebugHandlerImpl {
}

impl EntityDebugHandler for EntityDebugHandlerImpl {
fn query(&self, query: EntityQuery) -> Result<EntityDataValue, RpcError> {
self.query_impl(query)
fn query(&self, query: EntityQueryWithParams) -> Result<EntityDataValue, RpcError> {
let store = if query.use_cold_storage {
self.cold_store.clone().ok_or_else(|| {
RpcError::new_internal_error(None, "Cold storage is not available".to_string())
})?
} else {
self.hot_store.clone()
};
self.query_impl(store, query.query)
.map_err(|err| RpcError::new_internal_error(None, format!("{:?}", err)))
}
}
Expand Down
4 changes: 3 additions & 1 deletion nearcore/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,7 @@ pub fn start_with_config_and_synchronization(
state_sync_dumper.start()?;

let hot_store = storage.get_hot_store();
let cold_store = storage.get_cold_store();

let mut rpc_servers = Vec::new();
let network_actor = PeerManagerActor::spawn(
Expand All @@ -465,7 +466,8 @@ pub fn start_with_config_and_synchronization(
let entity_debug_handler = EntityDebugHandlerImpl {
epoch_manager: view_epoch_manager,
runtime: view_runtime,
store: hot_store,
hot_store,
cold_store,
};
rpc_servers.extend(near_jsonrpc::start_http(
rpc_config,
Expand Down
4 changes: 1 addition & 3 deletions tools/debug-ui/src/CurrentPeersView.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,9 +272,7 @@ export const CurrentPeersView = ({ addr }: NetworkInfoViewProps) => {
<td>
{peer.nonce}
<br />
{formatDurationInMillis(
Date.now() - peer.nonce * 1000
)}
{formatDurationInMillis(Date.now() - peer.nonce * 1000)}
</td>
<td>
{formatDurationInMillis(
Expand Down
Loading

0 comments on commit 328a8da

Please sign in to comment.