refactor(prune): segment trait, receipts impl (paradigmxyz#4887)

ivan-kyu · Oct 5, 2023 · 1e7d028 · 1e7d028
1 parent 4dceabf
commit 1e7d028
Show file tree

Hide file tree

Showing 43 changed files with 633 additions and 428 deletions.
diff --git a/bin/reth/src/args/pruning_args.rs b/bin/reth/src/args/pruning_args.rs
@@ -23,7 +23,7 @@ impl PruningArgs {
         Ok(if self.full {
             Some(PruneConfig {
                 block_interval: 5,
-                parts: PruneModes {
+                segments: PruneModes {
                     sender_recovery: Some(PruneMode::Full),
                     transaction_lookup: None,
                     receipts: chain_spec

diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs
@@ -193,7 +193,7 @@ impl ImportCommand {
                         .clean_threshold
                         .max(config.stages.account_hashing.clean_threshold)
                         .max(config.stages.storage_hashing.clean_threshold),
-                    config.prune.map(|prune| prune.parts).unwrap_or_default(),
+                    config.prune.map(|prune| prune.segments).unwrap_or_default(),
                 )),
             )
             .build(db, self.chain.clone());

diff --git a/bin/reth/src/debug_cmd/execution.rs b/bin/reth/src/debug_cmd/execution.rs
@@ -145,7 +145,7 @@ impl Command {
                         .clean_threshold
                         .max(stage_conf.account_hashing.clean_threshold)
                         .max(stage_conf.storage_hashing.clean_threshold),
-                    config.prune.as_ref().map(|prune| prune.parts.clone()).unwrap_or_default(),
+                    config.prune.as_ref().map(|prune| prune.segments.clone()).unwrap_or_default(),
                 )),
             )
             .build(db, self.chain.clone());

diff --git a/bin/reth/src/node/events.rs b/bin/reth/src/node/events.rs
@@ -163,8 +163,8 @@ impl NodeState {
 
     fn handle_pruner_event(&self, event: PrunerEvent) {
         match event {
-            PrunerEvent::Finished { tip_block_number, elapsed, parts } => {
-                info!(tip_block_number, ?elapsed, ?parts, "Pruner finished");
+            PrunerEvent::Finished { tip_block_number, elapsed, segments } => {
+                info!(tip_block_number, ?elapsed, ?segments, "Pruner finished");
             }
         }
     }

diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs
@@ -286,7 +286,7 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
         let tree = BlockchainTree::new(
             tree_externals,
             BlockchainTreeConfig::default(),
-            prune_config.clone().map(|config| config.parts),
+            prune_config.clone().map(|config| config.segments),
         )?
         .with_sync_metrics_tx(metrics_tx.clone());
         let canon_state_notification_sender = tree.canon_state_notification_sender();
@@ -461,7 +461,7 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
                 db.clone(),
                 self.chain.clone(),
                 prune_config.block_interval,
-                prune_config.parts,
+                prune_config.segments,
                 self.chain.prune_delete_limit,
                 highest_snapshots_rx,
             );
@@ -829,7 +829,7 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
 
         let factory = factory.with_stack_config(stack_config);
 
-        let prune_modes = prune_config.map(|prune| prune.parts).unwrap_or_default();
+        let prune_modes = prune_config.map(|prune| prune.segments).unwrap_or_default();
 
         let header_mode =
             if continuous { HeaderSyncMode::Continuous } else { HeaderSyncMode::Tip(tip_rx) };

diff --git a/bin/reth/src/stage/run.rs b/bin/reth/src/stage/run.rs
@@ -205,7 +205,7 @@ impl Command {
                                 max_cumulative_gas: None,
                             },
                             config.stages.merkle.clean_threshold,
-                            config.prune.map(|prune| prune.parts).unwrap_or_default(),
+                            config.prune.map(|prune| prune.segments).unwrap_or_default(),
                         )),
                         None,
                     )

diff --git a/book/run/config.md b/book/run/config.md
@@ -335,8 +335,8 @@ nanos = 0
 
 The prune section configures the pruning configuration.
 
-You can configure the pruning of different parts of the data independently of others.
-For any unspecified parts, the default setting is no pruning.
+You can configure the pruning of different segments of the data independently of others.
+For any unspecified segments, the default setting is no pruning.
 
 ### Default config
 

diff --git a/book/run/pruning.md b/book/run/pruning.md
@@ -57,10 +57,10 @@ You can track the growth of Reth archive node size with our
 
 ### Pruned Node
 
-Different parts take up different amounts of disk space.
-If pruned fully, this is the total freed space you'll get, per part:
+Different segments take up different amounts of disk space.
+If pruned fully, this is the total freed space you'll get, per segment:
 
-| Part               | Size  |
+| Segment              | Size  |
 |--------------------|-------|
 | Sender Recovery    | 70GB  |
 | Transaction Lookup | 140GB |
@@ -96,7 +96,7 @@ Meaning, it prunes:
 is completed, so the disk space is reclaimed slowly.
 - Receipts up to the last 128 blocks, preserving all receipts with the logs from Beacon Deposit Contract
 
-Given the aforementioned part sizes, we get the following full node size:
+Given the aforementioned segment sizes, we get the following full node size:
 ```text
 Archive Node - Receipts - AccountHistory - StorageHistory = Full Node
 ```
@@ -106,15 +106,15 @@ Archive Node - Receipts - AccountHistory - StorageHistory = Full Node
 
 ## RPC support
 
-As it was mentioned in the [pruning configuration chapter](./config.md#the-prune-section), there are several parts
-which can be pruned independently of each other:
+As it was mentioned in the [pruning configuration chapter](./config.md#the-prune-section), there are several segments which can be pruned
+independently of each other:
 - Sender Recovery
 - Transaction Lookup
 - Receipts
 - Account History
 - Storage History
 
-Pruning of each of these parts disables different RPC methods, because the historical data or lookup indexes
+Pruning of each of these segments disables different RPC methods, because the historical data or lookup indexes
 become unavailable.
 
 ### Full Node
@@ -140,7 +140,7 @@ The following tables describe RPC methods available in the full node.
 
 #### `eth` namespace
 
-| RPC / Part                                | Note                                                     |
+| RPC / Segment                             | Note                                                     |
 |-------------------------------------------|----------------------------------------------------------|
 | `eth_accounts`                            |                                                          |
 | `eth_blockNumber`                         |                                                          |
@@ -188,15 +188,15 @@ The following tables describe RPC methods available in the full node.
 
 #### `net` namespace
 
-| RPC / Part      |
+| RPC / Segment   |
 |-----------------|
 | `net_listening` |
 | `net_peerCount` |
 | `net_version`   |
 
 #### `trace` namespace
 
-| RPC / Part                      | Note                         |
+| RPC / Segment                   | Note                         |
 |---------------------------------|------------------------------|
 | `trace_block`                   | Only for the last 128 blocks |
 | `trace_call`                    | Only for the last 128 blocks |
@@ -209,7 +209,7 @@ The following tables describe RPC methods available in the full node.
 
 #### `txpool` namespace
 
-| RPC / Part           |
+| RPC / Segment        |
 |----------------------|
 | `txpool_content`     |
 | `txpool_contentFrom` |
@@ -219,13 +219,13 @@ The following tables describe RPC methods available in the full node.
 
 ### Pruned Node
 
-The following tables describe the requirements for prune parts, per RPC method:
-- ✅ – if the part is pruned, the RPC method still works
-- ❌ - if the part is pruned, the RPC method doesn't work anymore
+The following tables describe the requirements for prune segments, per RPC method:
+- ✅ – if the segment is pruned, the RPC method still works
+- ❌ - if the segment is pruned, the RPC method doesn't work anymore
 
 #### `debug` namespace
 
-| RPC / Part                 | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
+| RPC / Segment              | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
 |----------------------------|-----------------|--------------------|----------|-----------------|-----------------|
 | `debug_getRawBlock`        | ✅               | ✅                  | ✅        | ✅               | ✅               |
 | `debug_getRawHeader`       | ✅               | ✅                  | ✅        | ✅               | ✅               |
@@ -241,7 +241,7 @@ The following tables describe the requirements for prune parts, per RPC method:
 
 #### `eth` namespace
 
-| RPC / Part                                | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
+| RPC / Segment                             | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
 |-------------------------------------------|-----------------|--------------------|----------|-----------------|-----------------|
 | `eth_accounts`                            | ✅               | ✅                  | ✅        | ✅               | ✅               |
 | `eth_blockNumber`                         | ✅               | ✅                  | ✅        | ✅               | ✅               |
@@ -289,15 +289,15 @@ The following tables describe the requirements for prune parts, per RPC method:
 
 #### `net` namespace
 
-| RPC / Part      | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
+| RPC / Segment   | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
 |-----------------|-----------------|--------------------|----------|-----------------|-----------------|
 | `net_listening` | ✅               | ✅                  | ✅        | ✅               | ✅               |
 | `net_peerCount` | ✅               | ✅                  | ✅        | ✅               | ✅               |
 | `net_version`   | ✅               | ✅                  | ✅        | ✅               | ✅               |
 
 #### `trace` namespace
 
-| RPC / Part                      | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
+| RPC / Segment                   | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
 |---------------------------------|-----------------|--------------------|----------|-----------------|-----------------|
 | `trace_block`                   | ✅               | ✅                  | ✅        | ❌               | ❌               |
 | `trace_call`                    | ✅               | ✅                  | ✅        | ❌               | ❌               |
@@ -310,7 +310,7 @@ The following tables describe the requirements for prune parts, per RPC method:
 
 #### `txpool` namespace
 
-| RPC / Part           | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
+| RPC / Segment        | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
 |----------------------|-----------------|--------------------|----------|-----------------|-----------------|
 | `txpool_content`     | ✅               | ✅                  | ✅        | ✅               | ✅               |
 | `txpool_contentFrom` | ✅               | ✅                  | ✅        | ✅               | ✅               |

diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs
@@ -292,12 +292,13 @@ pub struct PruneConfig {
     /// Minimum pruning interval measured in blocks.
     pub block_interval: usize,
     /// Pruning configuration for every part of the data that can be pruned.
-    pub parts: PruneModes,
+    #[serde(alias = "parts")]
+    pub segments: PruneModes,
 }
 
 impl Default for PruneConfig {
     fn default() -> Self {
-        Self { block_interval: 5, parts: PruneModes::none() }
+        Self { block_interval: 5, segments: PruneModes::none() }
     }
 }
 
@@ -338,7 +339,8 @@ mod tests {
     // ensures config deserialization is backwards compatible
     #[test]
     fn test_backwards_compatibility() {
-        let alpha_0_0_8 = r"#[stages.headers]
+        let alpha_0_0_8 = r"#
+[stages.headers]
 downloader_max_concurrent_requests = 100
 downloader_min_concurrent_requests = 5
 downloader_max_buffered_responses = 100
@@ -422,8 +424,33 @@ nanos = 0
 [sessions.protocol_breach_request_timeout]
 secs = 120
 nanos = 0
-#";
 
+[prune]
+block_interval = 5
+
+[prune.parts]
+sender_recovery = { distance = 16384 }
+transaction_lookup = 'full'
+receipts = { before = 1920000 }
+account_history = { distance = 16384 }
+storage_history = { distance = 16384 }
+[prune.parts.receipts_log_filter]
+'0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48' = { before = 17000000 }
+'0xdac17f958d2ee523a2206206994597c13d831ec7' = { distance = 1000 }
+#";
         let _conf: Config = toml::from_str(alpha_0_0_8).unwrap();
+
+        let alpha_0_0_11 = r"#
+[prune.segments]
+sender_recovery = { distance = 16384 }
+transaction_lookup = 'full'
+receipts = { before = 1920000 }
+account_history = { distance = 16384 }
+storage_history = { distance = 16384 }
+[prune.segments.receipts_log_filter]
+'0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48' = { before = 17000000 }
+'0xdac17f958d2ee523a2206206994597c13d831ec7' = { distance = 1000 }
+#";
+        let _conf: Config = toml::from_str(alpha_0_0_11).unwrap();
     }
 }
diff --git a/crates/consensus/beacon/src/engine/hooks/prune.rs b/crates/consensus/beacon/src/engine/hooks/prune.rs
@@ -174,7 +174,7 @@ struct Metrics {
 impl From<PrunerError> for EngineHookError {
     fn from(err: PrunerError) -> Self {
         match err {
-            PrunerError::PrunePart(_) | PrunerError::InconsistentData(_) => {
+            PrunerError::PruneSegment(_) | PrunerError::InconsistentData(_) => {
                 EngineHookError::Internal(Box::new(err))
             }
             PrunerError::Interface(err) => err.into(),

diff --git a/crates/consensus/beacon/src/engine/hooks/snapshot.rs b/crates/consensus/beacon/src/engine/hooks/snapshot.rs
@@ -84,7 +84,7 @@ impl<DB: Database + 'static> SnapshotHook<DB> {
 
                 let targets = snapshotter.get_snapshot_targets(finalized_block_number)?;
 
-                // Check if the snapshotting of any parts has been requested.
+                // Check if the snapshotting of any data has been requested.
                 if targets.any() {
                     let (tx, rx) = oneshot::channel();
                     self.task_spawner.spawn_critical_blocking(

diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs
@@ -1,4 +1,4 @@
-use reth_primitives::{BlockNumHash, Bloom, PrunePartError, B256};
+use reth_primitives::{BlockNumHash, Bloom, PruneSegmentError, B256};
 use thiserror::Error;
 
 /// Transaction validation errors
@@ -76,9 +76,9 @@ pub enum BlockExecutionError {
     /// Validation error, transparently wrapping `BlockValidationError`
     #[error(transparent)]
     Validation(#[from] BlockValidationError),
-    /// Pruning error, transparently wrapping `PrunePartError`
+    /// Pruning error, transparently wrapping `PruneSegmentError`
     #[error(transparent)]
-    Pruning(#[from] PrunePartError),
+    Pruning(#[from] PruneSegmentError),
     /// Error representing a provider error
     #[error("Provider error")]
     ProviderError,

diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs
@@ -78,7 +78,7 @@ pub use net::{
 };
 pub use peer::{PeerId, WithPeerId};
 pub use prune::{
-    PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, PruneProgress,
+    PruneCheckpoint, PruneMode, PruneModes, PruneProgress, PruneSegment, PruneSegmentError,
     ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE,
 };
 pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef, Receipts};

diff --git a/crates/primitives/src/prune/checkpoint.rs b/crates/primitives/src/prune/checkpoint.rs
@@ -6,8 +6,7 @@ use reth_codecs::{main_codec, Compact};
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 #[cfg_attr(test, derive(Default))]
 pub struct PruneCheckpoint {
-    /// Highest pruned block number.
-    /// If it's [None], the pruning for block `0` is not finished yet.
+    /// Highest pruned block number. If it's [None], the pruning for block `0` is not finished yet.
     pub block_number: Option<BlockNumber>,
     /// Highest pruned transaction number, if applicable.
     pub tx_number: Option<TxNumber>,

diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs
@@ -1,12 +1,12 @@
 mod checkpoint;
 mod mode;
-mod part;
+mod segment;
 mod target;
 
 use crate::{Address, BlockNumber};
 pub use checkpoint::PruneCheckpoint;
 pub use mode::PruneMode;
-pub use part::{PrunePart, PrunePartError};
+pub use segment::{PruneSegment, PruneSegmentError};
 use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE};
@@ -40,7 +40,7 @@ impl ReceiptsLogPruneConfig {
         &self,
         tip: BlockNumber,
         pruned_block: Option<BlockNumber>,
-    ) -> Result<BTreeMap<BlockNumber, Vec<&Address>>, PrunePartError> {
+    ) -> Result<BTreeMap<BlockNumber, Vec<&Address>>, PruneSegmentError> {
         let mut map = BTreeMap::new();
         let pruned_block = pruned_block.unwrap_or_default();
 
@@ -53,7 +53,7 @@ impl ReceiptsLogPruneConfig {
             // Reminder, that we increment because the [`BlockNumber`] key of the new map should be
             // viewed as `PruneMode::Before(block)`
             let block = (pruned_block + 1).max(
-                mode.prune_target_block(tip, MINIMUM_PRUNING_DISTANCE, PrunePart::ContractLogs)?
+                mode.prune_target_block(tip, MINIMUM_PRUNING_DISTANCE, PruneSegment::ContractLogs)?
                     .map(|(block, _)| block)
                     .unwrap_or_default() +
                     1,
@@ -69,15 +69,17 @@ impl ReceiptsLogPruneConfig {
         &self,
         tip: BlockNumber,
         pruned_block: Option<BlockNumber>,
-    ) -> Result<Option<BlockNumber>, PrunePartError> {
+    ) -> Result<Option<BlockNumber>, PruneSegmentError> {
         let pruned_block = pruned_block.unwrap_or_default();
         let mut lowest = None;
 
         for (_, mode) in self.0.iter() {
             if let PruneMode::Distance(_) = mode {
-                if let Some((block, _)) =
-                    mode.prune_target_block(tip, MINIMUM_PRUNING_DISTANCE, PrunePart::ContractLogs)?
-                {
+                if let Some((block, _)) = mode.prune_target_block(
+                    tip,
+                    MINIMUM_PRUNING_DISTANCE,
+                    PruneSegment::ContractLogs,
+                )? {
                     lowest = Some(lowest.unwrap_or(u64::MAX).min(block));
                 }
             }