feat: epoch analysis tool (near#11343)

Tool to analyse epoch infos in two modes:
* `check-consistency` - regenerate next next epoch info based on two
previous epochs and check that it matches the epoch info stored in DB;
* `backtest` - regenerate epoch infos with existing proposals, rewards
and kickouts as if `PROTOCOL_VERSION` was always in place.

The `backtest` was used to estimate new algorithm for chunk producer
shard assignments and showed that on average there is only one state
sync happening, if we use start epoch height >= 545. Epoch info for 544
can't be retrieved for some reason, see near#11477.

Consistency check revealed that some epochs in the past can't be
replayed, see near#11476.
Expected output of it, note that epoch T generates epoch T+2:
```
$ neard view-state epoch-analysis --start-height 1359 check-consistency
HEIGHT | VERSION | STATE SYNCS
  1361 |      53 |           0
  1362 |      53 |           8
  1363 |      54 |           8
  1364 |      54 |           8
  1365 |      54 |          24
  1366 |      54 |          12
...
```

chain/epoch-manager/src/lib.rs

@@ -1,5 +1,4 @@
 use crate::metrics::{PROTOCOL_VERSION_NEXT, PROTOCOL_VERSION_VOTES};
-use crate::proposals::proposals_to_epoch_info;
 use crate::types::EpochInfoAggregator;
 use near_cache::SyncLruCache;
 use near_chain_configs::GenesisConfig;
@@ -35,6 +34,7 @@ use tracing::{debug, warn};
 use types::BlockHeaderInfo;
 
 pub use crate::adapter::EpochManagerAdapter;
+pub use crate::proposals::proposals_to_epoch_info;
 pub use crate::reward_calculator::RewardCalculator;
 pub use crate::reward_calculator::NUM_SECONDS_IN_A_YEAR;
 pub use crate::types::RngSeed;
@@ -1708,9 +1708,16 @@ impl EpochManager {
         Ok(ShardConfig::new(epoch_config))
     }
 
+    pub fn get_config_for_protocol_version(
+        &self,
+        protocol_version: ProtocolVersion,
+    ) -> Result<EpochConfig, EpochError> {
+        Ok(self.config.for_protocol_version(protocol_version))
+    }
+
     pub fn get_epoch_config(&self, epoch_id: &EpochId) -> Result<EpochConfig, EpochError> {
         let protocol_version = self.get_epoch_info(epoch_id)?.protocol_version();
-        Ok(self.config.for_protocol_version(protocol_version))
+        self.get_config_for_protocol_version(protocol_version)
     }
 
     pub fn get_shard_layout(&self, epoch_id: &EpochId) -> Result<ShardLayout, EpochError> {

chain/epoch-manager/src/proposals.rs

@@ -56,7 +56,7 @@ pub fn proposals_to_epoch_info(
         AliasValidatorSelectionAlgorithm,
         prev_prev_epoch_protocol_version
     ) {
-        return crate::validator_selection::proposals_to_epoch_info(
+        crate::validator_selection::proposals_to_epoch_info(
             epoch_config,
             rng_seed,
             prev_epoch_info,
@@ -66,9 +66,9 @@ pub fn proposals_to_epoch_info(
             minted_amount,
             protocol_version,
             use_stable_shard_assignment,
-        );
+        )
     } else {
-        return old_validator_selection::proposals_to_epoch_info(
+        old_validator_selection::proposals_to_epoch_info(
             epoch_config,
             rng_seed,
             prev_epoch_info,
@@ -77,7 +77,7 @@ pub fn proposals_to_epoch_info(
             validator_reward,
             minted_amount,
             protocol_version,
-        );
+        )
     }
 }
 

core/primitives/src/epoch_manager.rs

@@ -948,6 +948,16 @@ pub mod epoch_info {
             }
         }
 
+        #[inline]
+        pub fn chunk_producers_settlement_mut(&mut self) -> &mut Vec<Vec<ValidatorId>> {
+            match self {
+                Self::V1(v1) => &mut v1.chunk_producers_settlement,
+                Self::V2(v2) => &mut v2.chunk_producers_settlement,
+                Self::V3(v3) => &mut v3.chunk_producers_settlement,
+                Self::V4(v4) => &mut v4.chunk_producers_settlement,
+            }
+        }
+
         #[inline]
         pub fn validator_kickout(&self) -> &HashMap<AccountId, ValidatorKickoutReason> {
             match self {
@@ -1127,6 +1137,15 @@ pub mod epoch_info {
             }
         }
 
+        #[inline]
+        pub fn rng_seed(&self) -> RngSeed {
+            match self {
+                Self::V1(_) | Self::V2(_) => Default::default(),
+                Self::V3(v3) => v3.rng_seed,
+                Self::V4(v4) => v4.rng_seed,
+            }
+        }
+
         pub fn sample_block_producer(&self, height: BlockHeight) -> ValidatorId {
             match &self {
                 Self::V1(v1) => {

tools/state-viewer/src/cli.rs

@@ -10,6 +10,7 @@ use near_primitives::hash::CryptoHash;
 use near_primitives::sharding::ChunkHash;
 use near_primitives::trie_key::col;
 use near_primitives::types::{BlockHeight, ShardId};
+use near_primitives_core::types::EpochHeight;
 use near_store::{Mode, NodeStorage, Store, Temperature};
 use nearcore::{load_config, NearConfig};
 use std::path::{Path, PathBuf};
@@ -65,6 +66,9 @@ pub enum StateViewerSubCommand {
     /// Print `EpochInfo` of an epoch given by `--epoch_id` or by `--epoch_height`.
     #[clap(alias = "epoch_info")]
     EpochInfo(EpochInfoCmd),
+    /// Regenerates epoch info based on previous epoch.
+    #[clap(alias = "epoch_analysis")]
+    EpochAnalysis(EpochAnalysisCmd),
     /// Looks up a certain partial chunk.
     #[clap(alias = "partial_chunks")]
     PartialChunks(PartialChunksCmd),
@@ -152,6 +156,7 @@ impl StateViewerSubCommand {
             StateViewerSubCommand::DumpStateRedis(cmd) => cmd.run(home_dir, near_config, store),
             StateViewerSubCommand::DumpTx(cmd) => cmd.run(home_dir, near_config, store),
             StateViewerSubCommand::EpochInfo(cmd) => cmd.run(near_config, store),
+            StateViewerSubCommand::EpochAnalysis(cmd) => cmd.run(near_config, store),
             StateViewerSubCommand::PartialChunks(cmd) => cmd.run(near_config, store),
             StateViewerSubCommand::Receipts(cmd) => cmd.run(near_config, store),
             StateViewerSubCommand::Replay(cmd) => cmd.run(near_config, store),
@@ -495,6 +500,36 @@ impl EpochInfoCmd {
     }
 }
 
+#[derive(clap::Args)]
+pub struct EpochAnalysisCmd {
+    /// Start height of the epochs to analyse.
+    #[clap(long)]
+    start_height: EpochHeight,
+    /// Epoch analysis mode.
+    #[clap(subcommand)]
+    mode: EpochAnalysisMode,
+}
+
+#[derive(clap::Subcommand)]
+pub enum EpochAnalysisMode {
+    /// Regenerate epoch infos based on previous epoch, assert that epoch info
+    /// generation is replayable.
+    /// TODO (#11476): doesn't work when start epoch height is <= 1053 because
+    /// it will try to generate epoch with height 1055 and fail.
+    CheckConsistency,
+    /// Generate epoch infos as if latest `PROTOCOL_VERSION` was used since the
+    /// start epoch height.
+    /// TODO (#11477): doesn't work for start epoch height <= 544 because of
+    /// `EpochOutOfBounds` error.
+    Backtest,
+}
+
+impl EpochAnalysisCmd {
+    pub fn run(self, near_config: NearConfig, store: Store) {
+        print_epoch_analysis(self.start_height, self.mode, near_config, store);
+    }
+}
+
 #[derive(clap::Parser)]
 pub struct PartialChunksCmd {
     #[clap(long)]