feat(storage, tree): respect Transaction Lookup pruning in the bloc…

…kchain tree (paradigmxyz#4410)
jamesatomc · Sep 1, 2023 · e66e3e3 · e66e3e3
1 parent 39a6fa1
commit e66e3e3
Show file tree

Hide file tree

Showing 29 changed files with 118 additions and 49 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -156,3 +156,4 @@ c-kzg = { git = "https://github.com/ethereum/c-kzg-4844" }
 ### misc-testing
 proptest = "1.0"
 arbitrary = "1.1"
+assert_matches = "1.5.0"
diff --git a/bin/reth/src/debug_cmd/in_memory_merkle.rs b/bin/reth/src/debug_cmd/in_memory_merkle.rs
@@ -193,7 +193,7 @@ impl Command {
         let provider_rw = factory.provider_rw()?;
 
         // Insert block, state and hashes
-        provider_rw.insert_block(block.clone(), None)?;
+        provider_rw.insert_block(block.clone(), None, None)?;
         block_state.write_to_db(provider_rw.tx_ref(), block.number)?;
         let storage_lists = provider_rw.changed_storages_with_range(block.number..=block.number)?;
         let storages = provider_rw.plainstate_storages(storage_lists)?;

diff --git a/bin/reth/src/debug_cmd/merkle.rs b/bin/reth/src/debug_cmd/merkle.rs
@@ -185,7 +185,7 @@ impl Command {
                     continue
                 }
             };
-            provider_rw.insert_block(sealed_block.block, Some(sealed_block.senders))?;
+            provider_rw.insert_block(sealed_block.block, Some(sealed_block.senders), None)?;
         }
 
         // Check if any of hashing or merkle stages aren't on the same block number as

diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs
@@ -267,6 +267,9 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
         let metrics_listener = MetricsListener::new(metrics_rx);
         ctx.task_executor.spawn_critical("metrics listener task", metrics_listener);
 
+        let prune_config =
+            self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone());
+
         // configure blockchain tree
         let tree_externals = TreeExternals::new(
             db.clone(),
@@ -284,6 +287,7 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
                 tree_externals,
                 canon_state_notification_sender.clone(),
                 tree_config,
+                prune_config.clone().map(|config| config.parts),
             )?
             .with_sync_metrics_tx(metrics_tx.clone()),
         );
@@ -365,9 +369,6 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
             None
         };
 
-        let prune_config =
-            self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone());
-
         // Configure the pipeline
         let (mut pipeline, client) = if self.dev.dev {
             info!(target: "reth::cli", "Starting Reth in dev mode");

diff --git a/crates/blockchain-tree/Cargo.toml b/crates/blockchain-tree/Cargo.toml
@@ -40,7 +40,7 @@ reth-interfaces = { workspace = true, features = ["test-utils"] }
 reth-primitives = { workspace = true , features = ["test-utils"] }
 reth-provider = { workspace = true, features = ["test-utils"] }
 parking_lot.workspace = true
-assert_matches = "1.5"
+assert_matches.workspace = true
 tokio = { workspace = true, features = ["macros", "sync"] }
 
 [features]

diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs
@@ -16,7 +16,7 @@ use reth_interfaces::{
     Error,
 };
 use reth_primitives::{
-    BlockHash, BlockNumHash, BlockNumber, ForkBlock, Hardfork, Receipt, SealedBlock,
+    BlockHash, BlockNumHash, BlockNumber, ForkBlock, Hardfork, PruneModes, Receipt, SealedBlock,
     SealedBlockWithSenders, SealedHeader, U256,
 };
 use reth_provider::{
@@ -93,6 +93,7 @@ pub struct BlockchainTree<DB: Database, C: Consensus, EF: ExecutorFactory> {
     metrics: TreeMetrics,
     /// Metrics for sync stages.
     sync_metrics_tx: Option<MetricEventsSender>,
+    prune_modes: Option<PruneModes>,
 }
 
 /// A container that wraps chains and block indices to allow searching for block hashes across all
@@ -110,6 +111,7 @@ impl<DB: Database, C: Consensus, EF: ExecutorFactory> BlockchainTree<DB, C, EF>
         externals: TreeExternals<DB, C, EF>,
         canon_state_notification_sender: CanonStateNotificationSender,
         config: BlockchainTreeConfig,
+        prune_modes: Option<PruneModes>,
     ) -> Result<Self, Error> {
         let max_reorg_depth = config.max_reorg_depth();
 
@@ -145,6 +147,7 @@ impl<DB: Database, C: Consensus, EF: ExecutorFactory> BlockchainTree<DB, C, EF>
             canon_state_notification_sender,
             metrics: Default::default(),
             sync_metrics_tx: None,
+            prune_modes,
         })
     }
 
@@ -1048,7 +1051,11 @@ impl<DB: Database, C: Consensus, EF: ExecutorFactory> BlockchainTree<DB, C, EF>
         let (blocks, state) = chain.into_inner();
 
         provider
-            .append_blocks_with_post_state(blocks.into_blocks().collect(), state)
+            .append_blocks_with_post_state(
+                blocks.into_blocks().collect(),
+                state,
+                self.prune_modes.as_ref(),
+            )
             .map_err(|e| BlockExecutionError::CanonicalCommit { inner: e.to_string() })?;
 
         provider.commit()?;
@@ -1173,7 +1180,7 @@ mod tests {
         let factory = ProviderFactory::new(&db, MAINNET.clone());
         let provider = factory.provider_rw().unwrap();
 
-        provider.insert_block(genesis, None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
 
         // insert first 10 blocks
         for i in 0..10 {
@@ -1279,7 +1286,7 @@ mod tests {
         let config = BlockchainTreeConfig::new(1, 2, 3, 2);
         let (sender, mut canon_notif) = tokio::sync::broadcast::channel(10);
         let mut tree =
-            BlockchainTree::new(externals, sender, config).expect("failed to create tree");
+            BlockchainTree::new(externals, sender, config, None).expect("failed to create tree");
 
         // genesis block 10 is already canonical
         assert!(tree.make_canonical(&H256::zero()).is_ok());

diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml
@@ -46,4 +46,4 @@ reth-tracing = { path = "../../tracing" }
 reth-revm = { path = "../../revm" }
 reth-downloaders = { path = "../../net/downloaders" }
 
-assert_matches = "1.5"
+assert_matches.workspace = true
diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs
@@ -1880,7 +1880,9 @@ mod tests {
         BeaconForkChoiceUpdateError,
     };
     use assert_matches::assert_matches;
-    use reth_primitives::{stage::StageCheckpoint, ChainSpec, ChainSpecBuilder, H256, MAINNET};
+    use reth_primitives::{
+        stage::StageCheckpoint, ChainSpec, ChainSpecBuilder, PruneModes, H256, MAINNET,
+    };
     use reth_provider::{BlockWriter, ProviderFactory};
     use reth_rpc_types::engine::{
         ExecutionPayloadV1, ForkchoiceState, ForkchoiceUpdated, PayloadStatus,
@@ -2056,7 +2058,7 @@ mod tests {
         let factory = ProviderFactory::new(db, chain);
         let provider = factory.provider_rw().unwrap();
         blocks
-            .try_for_each(|b| provider.insert_block(b.clone(), None).map(|_| ()))
+            .try_for_each(|b| provider.insert_block(b.clone(), None, None).map(|_| ()))
             .expect("failed to insert");
         provider.commit().unwrap();
     }

diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs
@@ -463,7 +463,7 @@ where
         let config = BlockchainTreeConfig::new(1, 2, 3, 2);
         let (canon_state_notification_sender, _) = tokio::sync::broadcast::channel(3);
         let tree = ShareableBlockchainTree::new(
-            BlockchainTree::new(externals, canon_state_notification_sender, config)
+            BlockchainTree::new(externals, canon_state_notification_sender, config, None)
                 .expect("failed to create tree"),
         );
         let shareable_db = ProviderFactory::new(db.clone(), self.base_config.chain_spec.clone());

diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml
@@ -16,5 +16,5 @@ reth-provider.workspace = true
 [dev-dependencies]
 reth-interfaces = { workspace = true, features = ["test-utils"] }
 reth-provider = { workspace = true, features = ["test-utils"] }
-assert_matches = "1.5.0"
+assert_matches.workspace = true
 mockall = "0.11.3"
diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml
@@ -42,7 +42,7 @@ reth-db = { path = "../../storage/db", features = ["test-utils"] }
 reth-interfaces = { workspace = true, features = ["test-utils"] }
 reth-tracing = { path = "../../tracing" }
 
-assert_matches = "1.5.0"
+assert_matches.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 reth-rlp.workspace = true
 itertools.workspace = true

diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml
@@ -85,7 +85,7 @@ revm-primitives = { workspace = true, features = ["arbitrary"] }
 arbitrary = { workspace = true, features = ["derive"] }
 proptest.workspace = true
 proptest-derive = "0.3"
-assert_matches = "1.5.0"
+assert_matches.workspace = true
 toml = "0.7.4"
 
 # necessary so we don't hit a "undeclared 'std'": 

diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs
@@ -49,6 +49,11 @@ impl PruneMode {
             PruneMode::Before(n) => *n > block,
         }
     }
+
+    /// Returns true if the prune mode is [`PruneMode::Full`].
+    pub fn is_full(&self) -> bool {
+        matches!(self, Self::Full)
+    }
 }
 
 #[cfg(test)]

diff --git a/crates/prune/Cargo.toml b/crates/prune/Cargo.toml
@@ -34,4 +34,4 @@ reth-stages = { path = "../stages", features = ["test-utils"] }
 
 # misc
 
-assert_matches = "1.5.0"
+assert_matches.workspace = true
diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml
@@ -34,4 +34,4 @@ reth-rlp.workspace = true
 reth-interfaces = { workspace = true, features = ["test-utils"] }
 reth-provider = { workspace = true, features = ["test-utils"] }
 reth-payload-builder = { workspace = true, features = ["test-utils"] }
-assert_matches = "1.5.0"
+assert_matches.workspace = true
diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml
@@ -70,6 +70,6 @@ futures.workspace = true
 
 [dev-dependencies]
 jsonrpsee = { workspace = true, features = ["client"] }
-assert_matches = "1.5.0"
+assert_matches.workspace = true
 tempfile = "3.5.0"
 reth-interfaces = { workspace = true, features = ["test-utils"] }
diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml
@@ -61,7 +61,7 @@ reth-trie = { path = "../trie", features = ["test-utils"] }
 
 itertools.workspace = true
 tokio = { workspace = true, features = ["rt", "sync", "macros"] }
-assert_matches = "1.5.0"
+assert_matches.workspace = true
 rand.workspace = true
 paste = "1.0"
 

diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs
@@ -526,8 +526,8 @@ mod tests {
         let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap();
         let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice();
         let block = SealedBlock::decode(&mut block_rlp).unwrap();
-        provider.insert_block(genesis, None).unwrap();
-        provider.insert_block(block.clone(), None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
+        provider.insert_block(block.clone(), None, None).unwrap();
         provider.commit().unwrap();
 
         let previous_stage_checkpoint = ExecutionCheckpoint {
@@ -562,8 +562,8 @@ mod tests {
         let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap();
         let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice();
         let block = SealedBlock::decode(&mut block_rlp).unwrap();
-        provider.insert_block(genesis, None).unwrap();
-        provider.insert_block(block.clone(), None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
+        provider.insert_block(block.clone(), None, None).unwrap();
         provider.commit().unwrap();
 
         let previous_stage_checkpoint = ExecutionCheckpoint {
@@ -598,8 +598,8 @@ mod tests {
         let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap();
         let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice();
         let block = SealedBlock::decode(&mut block_rlp).unwrap();
-        provider.insert_block(genesis, None).unwrap();
-        provider.insert_block(block.clone(), None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
+        provider.insert_block(block.clone(), None, None).unwrap();
         provider.commit().unwrap();
 
         let previous_checkpoint = StageCheckpoint { block_number: 1, stage_checkpoint: None };
@@ -632,8 +632,8 @@ mod tests {
         let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap();
         let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice();
         let block = SealedBlock::decode(&mut block_rlp).unwrap();
-        provider.insert_block(genesis, None).unwrap();
-        provider.insert_block(block.clone(), None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
+        provider.insert_block(block.clone(), None, None).unwrap();
         provider.commit().unwrap();
 
         // insert pre state
@@ -742,8 +742,8 @@ mod tests {
         let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap();
         let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice();
         let block = SealedBlock::decode(&mut block_rlp).unwrap();
-        provider.insert_block(genesis, None).unwrap();
-        provider.insert_block(block.clone(), None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
+        provider.insert_block(block.clone(), None, None).unwrap();
         provider.commit().unwrap();
 
         // variables
@@ -820,8 +820,8 @@ mod tests {
         let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap();
         let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice();
         let block = SealedBlock::decode(&mut block_rlp).unwrap();
-        provider.insert_block(genesis, None).unwrap();
-        provider.insert_block(block.clone(), None).unwrap();
+        provider.insert_block(genesis, None, None).unwrap();
+        provider.insert_block(block.clone(), None, None).unwrap();
         provider.commit().unwrap();
 
         // variables
Original file line number	Diff line number	Diff line change
Expand Up		@@ -34,4 +34,4 @@ reth-stages = { path = "../stages", features = ["test-utils"] }

		# misc

		assert_matches = "1.5.0"
		assert_matches.workspace = true