feat: trait-based storage API (paradigmxyz#12616)

Co-authored-by: joshie <[email protected]>
CRossel87a · Nov 19, 2024 · 1e7189d · 1e7189d
1 parent 66a9d3e
commit 1e7189d
Show file tree

Hide file tree

Showing 36 changed files with 485 additions and 148 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bin/reth/src/commands/debug_cmd/execution.rs b/bin/reth/src/commands/debug_cmd/execution.rs
@@ -59,7 +59,7 @@ pub struct Command<C: ChainSpecParser> {
 }
 
 impl<C: ChainSpecParser<ChainSpec = ChainSpec>> Command<C> {
-    fn build_pipeline<N: ProviderNodeTypes<ChainSpec = C::ChainSpec>, Client>(
+    fn build_pipeline<N: ProviderNodeTypes<ChainSpec = C::ChainSpec> + CliNodeTypes, Client>(
         &self,
         config: &Config,
         client: Client,

diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml
@@ -32,6 +32,7 @@ reth-fs-util.workspace = true
 reth-network = { workspace = true, features = ["serde"] }
 reth-network-p2p.workspace = true
 reth-network-peers = { workspace = true, features = ["secp256k1"] }
+reth-node-api.workspace = true
 reth-node-builder.workspace = true
 reth-node-core.workspace = true
 reth-node-events.workspace = true

diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs
@@ -10,12 +10,16 @@ use reth_db::{init_db, open_db_read_only, DatabaseEnv};
 use reth_db_common::init::init_genesis;
 use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader};
 use reth_evm::noop::NoopBlockExecutorProvider;
+use reth_node_api::FullNodePrimitives;
 use reth_node_builder::{NodeTypesWithDBAdapter, NodeTypesWithEngine};
 use reth_node_core::{
     args::{DatabaseArgs, DatadirArgs},
     dirs::{ChainPath, DataDirPath},
 };
-use reth_provider::{providers::StaticFileProvider, ProviderFactory, StaticFileProviderFactory};
+use reth_provider::{
+    providers::{NodeTypesForProvider, StaticFileProvider},
+    ProviderFactory, StaticFileProviderFactory,
+};
 use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
 use reth_static_file::StaticFileProducer;
 use std::{path::PathBuf, sync::Arc};
@@ -191,5 +195,21 @@ impl AccessRights {
 
 /// Helper trait with a common set of requirements for the
 /// [`NodeTypes`](reth_node_builder::NodeTypes) in CLI.
-pub trait CliNodeTypes: NodeTypesWithEngine<ChainSpec: EthereumHardforks> {}
-impl<N> CliNodeTypes for N where N: NodeTypesWithEngine<ChainSpec: EthereumHardforks> {}
+pub trait CliNodeTypes:
+    NodeTypesWithEngine
+    + NodeTypesForProvider<
+        Primitives: FullNodePrimitives<
+            Block: reth_node_api::Block<Body = reth_primitives::BlockBody>,
+        >,
+    >
+{
+}
+impl<N> CliNodeTypes for N where
+    N: NodeTypesWithEngine
+        + NodeTypesForProvider<
+            Primitives: FullNodePrimitives<
+                Block: reth_node_api::Block<Body = reth_primitives::BlockBody>,
+            >,
+        >
+{
+}
diff --git a/crates/cli/commands/src/import.rs b/crates/cli/commands/src/import.rs
@@ -167,7 +167,7 @@ pub fn build_import_pipeline<N, C, E>(
     executor: E,
 ) -> eyre::Result<(Pipeline<N>, impl Stream<Item = NodeEvent>)>
 where
-    N: ProviderNodeTypes,
+    N: ProviderNodeTypes + CliNodeTypes,
     C: Consensus + 'static,
     E: BlockExecutorProvider,
 {

diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs
@@ -113,7 +113,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
         Ok(())
     }
 
-    fn build_pipeline<N: ProviderNodeTypes<ChainSpec = C::ChainSpec>>(
+    fn build_pipeline<N: ProviderNodeTypes<ChainSpec = C::ChainSpec> + CliNodeTypes>(
         self,
         config: Config,
         provider_factory: ProviderFactory<N>,

diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml
@@ -19,7 +19,9 @@ reth-rpc-layer.workspace = true
 reth-payload-builder = { workspace = true, features = ["test-utils"] }
 reth-payload-builder-primitives.workspace = true
 reth-payload-primitives.workspace = true
+reth-primitives.workspace = true
 reth-provider.workspace = true
+reth-node-api.workspace = true
 reth-node-builder = { workspace = true, features = ["test-utils"] }
 reth-tokio-util.workspace = true
 reth-stages-types.workspace = true

diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs
@@ -5,20 +5,20 @@ use std::sync::Arc;
 use node::NodeTestContext;
 use reth::{
     args::{DiscoveryArgs, NetworkArgs, RpcServerArgs},
-    builder::{NodeBuilder, NodeConfig, NodeHandle},
+    builder::{FullNodePrimitives, NodeBuilder, NodeConfig, NodeHandle},
     network::PeersHandleProvider,
     rpc::server_types::RpcModuleSelection,
     tasks::TaskManager,
 };
-use reth_chainspec::{EthChainSpec, EthereumHardforks};
+use reth_chainspec::EthChainSpec;
 use reth_db::{test_utils::TempDatabase, DatabaseEnv};
 use reth_engine_local::LocalPayloadAttributesBuilder;
 use reth_node_builder::{
     components::NodeComponentsBuilder, rpc::RethRpcAddOns, EngineNodeLauncher,
     FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypesWithDBAdapter,
     NodeTypesWithEngine, PayloadAttributesBuilder, PayloadTypes,
 };
-use reth_provider::providers::{BlockchainProvider, BlockchainProvider2};
+use reth_provider::providers::{BlockchainProvider, BlockchainProvider2, NodeTypesForProvider};
 use tracing::{span, Level};
 use wallet::Wallet;
 
@@ -53,12 +53,14 @@ pub async fn setup<N>(
     attributes_generator: impl Fn(u64) -> <<N as NodeTypesWithEngine>::Engine as PayloadTypes>::PayloadBuilderAttributes + Copy + 'static,
 ) -> eyre::Result<(Vec<NodeHelperType<N, N::AddOns>>, TaskManager, Wallet)>
 where
-    N: Default + Node<TmpNodeAdapter<N>> + NodeTypesWithEngine<ChainSpec: EthereumHardforks>,
+    N: Default + Node<TmpNodeAdapter<N>> + NodeTypesForProvider + NodeTypesWithEngine,
     N::ComponentsBuilder: NodeComponentsBuilder<
         TmpNodeAdapter<N>,
         Components: NodeComponents<TmpNodeAdapter<N>, Network: PeersHandleProvider>,
     >,
     N::AddOns: RethRpcAddOns<Adapter<N>>,
+    N::Primitives:
+        FullNodePrimitives<Block: reth_node_api::Block<Body = reth_primitives::BlockBody>>,
 {
     let tasks = TaskManager::current();
     let exec = tasks.executor();
@@ -120,7 +122,8 @@ pub async fn setup_engine<N>(
 where
     N: Default
         + Node<TmpNodeAdapter<N, BlockchainProvider2<NodeTypesWithDBAdapter<N, TmpDB>>>>
-        + NodeTypesWithEngine<ChainSpec: EthereumHardforks>,
+        + NodeTypesWithEngine
+        + NodeTypesForProvider,
     N::ComponentsBuilder: NodeComponentsBuilder<
         TmpNodeAdapter<N, BlockchainProvider2<NodeTypesWithDBAdapter<N, TmpDB>>>,
         Components: NodeComponents<
@@ -132,6 +135,8 @@ where
     LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
         <<N as NodeTypesWithEngine>::Engine as PayloadTypes>::PayloadAttributes,
     >,
+    N::Primitives:
+        FullNodePrimitives<Block: reth_node_api::Block<Body = reth_primitives::BlockBody>>,
 {
     let tasks = TaskManager::current();
     let exec = tasks.executor();

diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs
@@ -26,7 +26,7 @@ use reth_node_builder::{
 };
 use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService};
 use reth_primitives::EthPrimitives;
-use reth_provider::CanonStateSubscriptions;
+use reth_provider::{CanonStateSubscriptions, EthStorage};
 use reth_rpc::EthApi;
 use reth_tracing::tracing::{debug, info};
 use reth_transaction_pool::{
@@ -74,6 +74,7 @@ impl NodeTypes for EthereumNode {
     type Primitives = EthPrimitives;
     type ChainSpec = ChainSpec;
     type StateCommitment = MerklePatriciaTrie;
+    type Storage = EthStorage;
 }
 
 impl NodeTypesWithEngine for EthereumNode {
@@ -94,7 +95,13 @@ pub type EthereumAddOns<N> = RpcAddOns<
 
 impl<Types, N> Node<N> for EthereumNode
 where
-    Types: NodeTypesWithDB + NodeTypesWithEngine<Engine = EthEngineTypes, ChainSpec = ChainSpec>,
+    Types: NodeTypesWithDB
+        + NodeTypesWithEngine<
+            Engine = EthEngineTypes,
+            ChainSpec = ChainSpec,
+            Primitives = EthPrimitives,
+            Storage = EthStorage,
+        >,
     N: FullNodeTypes<Types = Types>,
 {
     type ComponentsBuilder = ComponentsBuilder<

diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs
@@ -45,10 +45,10 @@ use reth_node_ethereum::{
     EthEngineTypes, EthEvmConfig,
 };
 use reth_payload_builder::noop::NoopPayloadBuilderService;
-use reth_primitives::{Head, SealedBlockWithSenders};
+use reth_primitives::{EthPrimitives, Head, SealedBlockWithSenders};
 use reth_provider::{
     providers::{BlockchainProvider, StaticFileProvider},
-    BlockReader, ProviderFactory,
+    BlockReader, EthStorage, ProviderFactory,
 };
 use reth_tasks::TaskManager;
 use reth_transaction_pool::test_utils::{testing_pool, TestPool};
@@ -118,9 +118,10 @@ where
 pub struct TestNode;
 
 impl NodeTypes for TestNode {
-    type Primitives = ();
+    type Primitives = EthPrimitives;
     type ChainSpec = ChainSpec;
     type StateCommitment = reth_trie_db::MerklePatriciaTrie;
+    type Storage = EthStorage;
 }
 
 impl NodeTypesWithEngine for TestNode {
@@ -129,7 +130,14 @@ impl NodeTypesWithEngine for TestNode {
 
 impl<N> Node<N> for TestNode
 where
-    N: FullNodeTypes<Types: NodeTypesWithEngine<Engine = EthEngineTypes, ChainSpec = ChainSpec>>,
+    N: FullNodeTypes<
+        Types: NodeTypesWithEngine<
+            Engine = EthEngineTypes,
+            ChainSpec = ChainSpec,
+            Primitives = EthPrimitives,
+            Storage = EthStorage,
+        >,
+    >,
 {
     type ComponentsBuilder = ComponentsBuilder<
         N,

diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs
@@ -22,16 +22,19 @@ use reth_network::{
     NetworkHandle, NetworkManager,
 };
 use reth_node_api::{
-    FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter,
-    NodeTypesWithEngine,
+    FullNodePrimitives, FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes,
+    NodeTypesWithDBAdapter, NodeTypesWithEngine,
 };
 use reth_node_core::{
     cli::config::{PayloadBuilderConfig, RethTransactionPoolConfig},
     dirs::{ChainPath, DataDirPath},
     node_config::NodeConfig,
     primitives::Head,
 };
-use reth_provider::{providers::BlockchainProvider, ChainSpecProvider, FullProvider};
+use reth_provider::{
+    providers::{BlockchainProvider, NodeTypesForProvider},
+    ChainSpecProvider, FullProvider,
+};
 use reth_tasks::TaskExecutor;
 use reth_transaction_pool::{PoolConfig, TransactionPool};
 use revm_primitives::EnvKzgSettings;
@@ -240,7 +243,7 @@ where
     /// Configures the types of the node.
     pub fn with_types<T>(self) -> NodeBuilderWithTypes<RethFullAdapter<DB, T>>
     where
-        T: NodeTypesWithEngine<ChainSpec = ChainSpec>,
+        T: NodeTypesWithEngine<ChainSpec = ChainSpec> + NodeTypesForProvider,
     {
         self.with_types_and_provider()
     }
@@ -250,7 +253,7 @@ where
         self,
     ) -> NodeBuilderWithTypes<FullNodeTypesAdapter<NodeTypesWithDBAdapter<T, DB>, P>>
     where
-        T: NodeTypesWithEngine<ChainSpec = ChainSpec>,
+        T: NodeTypesWithEngine<ChainSpec = ChainSpec> + NodeTypesForProvider,
         P: FullProvider<NodeTypesWithDBAdapter<T, DB>>,
     {
         NodeBuilderWithTypes::new(self.config, self.database)
@@ -264,7 +267,7 @@ where
         node: N,
     ) -> NodeBuilderWithComponents<RethFullAdapter<DB, N>, N::ComponentsBuilder, N::AddOns>
     where
-        N: Node<RethFullAdapter<DB, N>, ChainSpec = ChainSpec>,
+        N: Node<RethFullAdapter<DB, N>, ChainSpec = ChainSpec> + NodeTypesForProvider,
     {
         self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons())
     }
@@ -301,7 +304,7 @@ where
     /// Configures the types of the node.
     pub fn with_types<T>(self) -> WithLaunchContext<NodeBuilderWithTypes<RethFullAdapter<DB, T>>>
     where
-        T: NodeTypesWithEngine<ChainSpec = ChainSpec>,
+        T: NodeTypesWithEngine<ChainSpec = ChainSpec> + NodeTypesForProvider,
     {
         WithLaunchContext { builder: self.builder.with_types(), task_executor: self.task_executor }
     }
@@ -313,7 +316,7 @@ where
         NodeBuilderWithTypes<FullNodeTypesAdapter<NodeTypesWithDBAdapter<T, DB>, P>>,
     >
     where
-        T: NodeTypesWithEngine<ChainSpec = ChainSpec>,
+        T: NodeTypesWithEngine<ChainSpec = ChainSpec> + NodeTypesForProvider,
         P: FullProvider<NodeTypesWithDBAdapter<T, DB>>,
     {
         WithLaunchContext {
@@ -332,7 +335,7 @@ where
         NodeBuilderWithComponents<RethFullAdapter<DB, N>, N::ComponentsBuilder, N::AddOns>,
     >
     where
-        N: Node<RethFullAdapter<DB, N>, ChainSpec = ChainSpec>,
+        N: Node<RethFullAdapter<DB, N>, ChainSpec = ChainSpec> + NodeTypesForProvider,
     {
         self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons())
     }
@@ -355,13 +358,15 @@ where
         >,
     >
     where
-        N: Node<RethFullAdapter<DB, N>, ChainSpec = ChainSpec>,
+        N: Node<RethFullAdapter<DB, N>, ChainSpec = ChainSpec> + NodeTypesForProvider,
         N::AddOns: RethRpcAddOns<
             NodeAdapter<
                 RethFullAdapter<DB, N>,
                 <N::ComponentsBuilder as NodeComponentsBuilder<RethFullAdapter<DB, N>>>::Components,
             >,
         >,
+        N::Primitives:
+            FullNodePrimitives<Block: reth_node_api::Block<Body = reth_primitives::BlockBody>>,
     {
         self.node(node).launch().await
     }
@@ -549,9 +554,11 @@ where
 impl<T, DB, CB, AO> WithLaunchContext<NodeBuilderWithComponents<RethFullAdapter<DB, T>, CB, AO>>
 where
     DB: Database + DatabaseMetrics + DatabaseMetadata + Clone + Unpin + 'static,
-    T: NodeTypesWithEngine<ChainSpec: EthereumHardforks + EthChainSpec>,
+    T: NodeTypesWithEngine + NodeTypesForProvider,
     CB: NodeComponentsBuilder<RethFullAdapter<DB, T>>,
     AO: RethRpcAddOns<NodeAdapter<RethFullAdapter<DB, T>, CB::Components>>,
+    T::Primitives:
+        FullNodePrimitives<Block: reth_node_api::Block<Body = reth_primitives::BlockBody>>,
 {
     /// Launches the node with the [`DefaultNodeLauncher`] that sets up engine API consensus and rpc
     pub async fn launch(