From a7431465fa9c04c25a1cedd9553f375eb3bfa7a8 Mon Sep 17 00:00:00 2001 From: evalir Date: Tue, 4 Jul 2023 06:15:52 -0400 Subject: [PATCH] chore(`crates`): deprecate `staged-sync` (#3564) --- Cargo.lock | 40 ------- Cargo.toml | 1 - bin/reth/Cargo.toml | 1 - bin/reth/src/chain/import.rs | 2 +- bin/reth/src/chain/init.rs | 2 +- bin/reth/src/debug_cmd/execution.rs | 2 +- .../src/utils => bin/reth/src}/init.rs | 1 + bin/reth/src/lib.rs | 1 + bin/reth/src/node/mod.rs | 2 +- bin/reth/src/stage/drop.rs | 2 +- codecov.yml | 1 - crates/net/network/Cargo.toml | 2 +- .../network/tests/it/clique}/clique.rs | 0 .../tests/it/clique}/clique_middleware.rs | 10 -- .../network/tests/it/clique}/mod.rs | 2 - .../sync.rs => net/network/tests/it/geth.rs} | 2 +- crates/net/network/tests/it/main.rs | 2 + crates/staged-sync/Cargo.toml | 101 ------------------ crates/staged-sync/src/lib.rs | 24 ----- crates/staged-sync/src/utils/mod.rs | 4 - docs/repo/layout.md | 1 - 21 files changed, 11 insertions(+), 192 deletions(-) rename {crates/staged-sync/src/utils => bin/reth/src}/init.rs (99%) rename crates/{staged-sync/src/test_utils => net/network/tests/it/clique}/clique.rs (100%) rename crates/{staged-sync/src/test_utils => net/network/tests/it/clique}/clique_middleware.rs (91%) rename crates/{staged-sync/src/test_utils => net/network/tests/it/clique}/mod.rs (74%) rename crates/{staged-sync/tests/sync.rs => net/network/tests/it/geth.rs} (98%) delete mode 100644 crates/staged-sync/Cargo.toml delete mode 100644 crates/staged-sync/src/lib.rs delete mode 100644 crates/staged-sync/src/utils/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 1fa48941cc26..50a480581a06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4996,7 +4996,6 @@ dependencies = [ "reth-rpc", "reth-rpc-builder", "reth-rpc-engine-api", - "reth-staged-sync", "reth-stages", "reth-tasks", "reth-tracing", @@ -5791,45 +5790,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "reth-staged-sync" -version = "0.1.0-alpha.1" -dependencies = [ - "assert_matches", - "async-trait", - "confy", - "enr", - "ethers-core", - "ethers-middleware", - "ethers-providers", - "ethers-signers", - "eyre", - "futures", - "hex", - "rand 0.8.5", - "reth-db", - "reth-discv4", - "reth-downloaders", - "reth-interfaces", - "reth-net-nat", - "reth-network", - "reth-network-api", - "reth-primitives", - "reth-provider", - "reth-staged-sync", - "reth-stages", - "reth-tracing", - "secp256k1", - "serde", - "serde_json", - "shellexpand", - "tempfile", - "thiserror", - "tokio", - "tracing", - "walkdir", -] - [[package]] name = "reth-stages" version = "0.1.0-alpha.1" diff --git a/Cargo.toml b/Cargo.toml index 52268e8ec0f5..11ba1a26fbb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,6 @@ members = [ "crates/rpc/rpc-engine-api", "crates/rpc/rpc-types", "crates/rpc/rpc-testing-util", - "crates/staged-sync", "crates/stages", "crates/storage/codecs", "crates/storage/db", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 0c1d7f881a74..7009807f75b0 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -16,7 +16,6 @@ reth-db = { path = "../../crates/storage/db", features = ["mdbx", "test-utils"] reth-provider = { workspace = true, features = ["test-utils"] } reth-revm = { path = "../../crates/revm" } reth-revm-inspectors = { path = "../../crates/revm/revm-inspectors" } -reth-staged-sync = { path = "../../crates/staged-sync" } reth-stages = { path = "../../crates/stages" } reth-interfaces = { workspace = true, features = ["test-utils", "clap"] } reth-transaction-pool = { workspace = true } diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index e18e7716a65a..38bed8996f81 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -1,5 +1,6 @@ use crate::{ dirs::{DataDirPath, MaybePlatformPath}, + init::init_genesis, node::events::{handle_events, NodeEvent}, version::SHORT_VERSION, }; @@ -18,7 +19,6 @@ use reth_downloaders::{ }; use reth_interfaces::consensus::Consensus; use reth_primitives::{stage::StageId, ChainSpec, H256}; -use reth_staged_sync::utils::init::init_genesis; use reth_stages::{ prelude::*, stages::{ diff --git a/bin/reth/src/chain/init.rs b/bin/reth/src/chain/init.rs index 3c2c73015a02..e25cbf8a2cfb 100644 --- a/bin/reth/src/chain/init.rs +++ b/bin/reth/src/chain/init.rs @@ -1,11 +1,11 @@ use crate::{ args::{utils::genesis_value_parser, DatabaseArgs}, dirs::{DataDirPath, MaybePlatformPath}, + init::init_genesis, }; use clap::Parser; use reth_db::init_db; use reth_primitives::ChainSpec; -use reth_staged_sync::utils::init::init_genesis; use std::sync::Arc; use tracing::info; diff --git a/bin/reth/src/debug_cmd/execution.rs b/bin/reth/src/debug_cmd/execution.rs index fda560715130..7a6f61334626 100644 --- a/bin/reth/src/debug_cmd/execution.rs +++ b/bin/reth/src/debug_cmd/execution.rs @@ -2,6 +2,7 @@ use crate::{ args::{get_secret_key, utils::genesis_value_parser, DatabaseArgs, NetworkArgs}, dirs::{DataDirPath, MaybePlatformPath}, + init::init_genesis, node::events, runner::CliContext, utils::get_single_header, @@ -24,7 +25,6 @@ use reth_network::NetworkHandle; use reth_network_api::NetworkInfo; use reth_primitives::{fs, stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, H256}; use reth_provider::{BlockExecutionWriter, ProviderFactory, StageCheckpointReader}; -use reth_staged_sync::utils::init::init_genesis; use reth_stages::{ sets::DefaultStages, stages::{ diff --git a/crates/staged-sync/src/utils/init.rs b/bin/reth/src/init.rs similarity index 99% rename from crates/staged-sync/src/utils/init.rs rename to bin/reth/src/init.rs index e3c1d70b2acd..30373b3100be 100644 --- a/crates/staged-sync/src/utils/init.rs +++ b/bin/reth/src/init.rs @@ -1,3 +1,4 @@ +//! Reth genesis initialization utility functions. use reth_db::{ cursor::DbCursorRO, database::{Database, DatabaseGAT}, diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index d42274e28657..46d2c76a16e0 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -30,6 +30,7 @@ pub mod config; pub mod db; pub mod debug_cmd; pub mod dirs; +pub mod init; pub mod node; pub mod p2p; pub mod prometheus_exporter; diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 18e13ce2cfb0..3362bfe3e86d 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -4,6 +4,7 @@ use crate::{ args::{get_secret_key, DebugArgs, NetworkArgs, RpcServerArgs}, dirs::DataDirPath, + init::init_genesis, prometheus_exporter, runner::CliContext, utils::get_single_header, @@ -46,7 +47,6 @@ use reth_provider::{ use reth_revm::Factory; use reth_revm_inspectors::stack::Hook; use reth_rpc_engine_api::EngineApi; -use reth_staged_sync::utils::init::init_genesis; use reth_stages::{ prelude::*, stages::{ diff --git a/bin/reth/src/stage/drop.rs b/bin/reth/src/stage/drop.rs index 2963c643867f..771dff1d2049 100644 --- a/bin/reth/src/stage/drop.rs +++ b/bin/reth/src/stage/drop.rs @@ -2,12 +2,12 @@ use crate::{ args::{utils::genesis_value_parser, DatabaseArgs, StageEnum}, dirs::{DataDirPath, MaybePlatformPath}, + init::{insert_genesis_header, insert_genesis_state}, utils::DbTool, }; use clap::Parser; use reth_db::{database::Database, open_db, tables, transaction::DbTxMut, DatabaseEnv}; use reth_primitives::{fs, stage::StageId, ChainSpec}; -use reth_staged_sync::utils::init::{insert_genesis_header, insert_genesis_state}; use std::sync::Arc; use tracing::info; diff --git a/codecov.yml b/codecov.yml index 6392d83a8d60..5bd75590b504 100644 --- a/codecov.yml +++ b/codecov.yml @@ -27,7 +27,6 @@ component_management: name: pipeline paths: - crates/stages/** - - crates/staged-sync/** - component_id: storage name: storage (db) paths: diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 8a62fec063a0..3e5195c1cc29 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -77,7 +77,7 @@ reth-tracing = { path = "../../tracing" } reth-transaction-pool = { workspace = true, features = ["test-utils"] } ethers-core = { workspace = true, default-features = false } -ethers-providers = { workspace = true, default-features = false } +ethers-providers = { workspace = true, default-features = false, features = ["ws"] } ethers-signers = { workspace = true, default-features = false } ethers-middleware = { workspace = true, default-features = false } diff --git a/crates/staged-sync/src/test_utils/clique.rs b/crates/net/network/tests/it/clique/clique.rs similarity index 100% rename from crates/staged-sync/src/test_utils/clique.rs rename to crates/net/network/tests/it/clique/clique.rs diff --git a/crates/staged-sync/src/test_utils/clique_middleware.rs b/crates/net/network/tests/it/clique/clique_middleware.rs similarity index 91% rename from crates/staged-sync/src/test_utils/clique_middleware.rs rename to crates/net/network/tests/it/clique/clique_middleware.rs index 315788c2e26c..ac06c7b9110e 100644 --- a/crates/staged-sync/src/test_utils/clique_middleware.rs +++ b/crates/net/network/tests/it/clique/clique_middleware.rs @@ -26,10 +26,6 @@ pub enum CliqueError { #[error("no genesis block returned from the provider")] NoGenesis, - /// No tip block returned from the provider - #[error("no tip block returned from the provider")] - NoTip, - /// Account was not successfully unlocked on the provider #[error("account was not successfully unlocked on the provider")] AccountNotUnlocked, @@ -94,12 +90,6 @@ pub trait CliqueMiddleware: Send + Sync + Middleware { Ok(()) } - /// Returns the chain tip of the [`Geth`](ethers_core::utils::Geth) instance by calling - /// geth's `eth_getBlock`. - async fn remote_tip_block(&self) -> Result, CliqueMiddlewareError> { - self.get_block(BlockNumber::Latest).await?.ok_or(CliqueError::NoTip) - } - /// Returns the genesis block of the [`Geth`](ethers_core::utils::Geth) instance by calling /// geth's `eth_getBlock`. async fn remote_genesis_block(&self) -> Result, CliqueMiddlewareError> { diff --git a/crates/staged-sync/src/test_utils/mod.rs b/crates/net/network/tests/it/clique/mod.rs similarity index 74% rename from crates/staged-sync/src/test_utils/mod.rs rename to crates/net/network/tests/it/clique/mod.rs index ff1105afade9..fd635c3cab2c 100644 --- a/crates/staged-sync/src/test_utils/mod.rs +++ b/crates/net/network/tests/it/clique/mod.rs @@ -1,5 +1,3 @@ -//! Common helpers for staged sync integration testing. - pub mod clique; pub mod clique_middleware; diff --git a/crates/staged-sync/tests/sync.rs b/crates/net/network/tests/it/geth.rs similarity index 98% rename from crates/staged-sync/tests/sync.rs rename to crates/net/network/tests/it/geth.rs index 66ef0f763bbb..3c9f8d41ff2c 100644 --- a/crates/staged-sync/tests/sync.rs +++ b/crates/net/network/tests/it/geth.rs @@ -1,3 +1,4 @@ +use crate::clique::{CliqueGethInstance, CliqueMiddleware}; use ethers_core::{ types::{transaction::eip2718::TypedTransaction, Eip1559TransactionRequest, H160, U64}, utils::Geth, @@ -10,7 +11,6 @@ use reth_network::{ use reth_network_api::Peers; use reth_primitives::{ChainSpec, Genesis, PeerId, SealedHeader}; use reth_provider::test_utils::NoopProvider; -use reth_staged_sync::test_utils::{CliqueGethInstance, CliqueMiddleware}; use secp256k1::SecretKey; use std::{net::SocketAddr, sync::Arc}; diff --git a/crates/net/network/tests/it/main.rs b/crates/net/network/tests/it/main.rs index ccba08a3a24a..106bf61301db 100644 --- a/crates/net/network/tests/it/main.rs +++ b/crates/net/network/tests/it/main.rs @@ -1,4 +1,6 @@ +mod clique; mod connect; +mod geth; mod requests; mod session; mod startup; diff --git a/crates/staged-sync/Cargo.toml b/crates/staged-sync/Cargo.toml deleted file mode 100644 index a88472583a06..000000000000 --- a/crates/staged-sync/Cargo.toml +++ /dev/null @@ -1,101 +0,0 @@ -[package] -name = "reth-staged-sync" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -description = "Puts together all the Reth stages in a unified abstraction" - -[dependencies] -# reth -reth-db = { path = "../../crates/storage/db", features = ["mdbx", "test-utils"] } -reth-discv4 = { path = "../../crates/net/discv4" } -reth-network-api = { workspace = true } -reth-network = { path = "../../crates/net/network", features = ["serde"] } -reth-downloaders = { path = "../../crates/net/downloaders" } -reth-primitives = { workspace = true } -reth-provider = { workspace = true, features = ["test-utils"] } -reth-net-nat = { path = "../../crates/net/nat" } -reth-stages = { path = "../stages" } -reth-interfaces = { workspace = true } - -# io -serde = "1.0" -serde_json = { workspace = true } - -# misc -walkdir = "2.3.2" -eyre = "0.6.8" -shellexpand = "3.0.0" -tracing = { workspace = true } - -# crypto -rand = { workspace = true, optional = true } -secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] } - -# errors -thiserror = { workspace = true } - -# enr -enr = { version = "0.8.1", features = ["serde", "rust-secp256k1"], optional = true } - -# ethers -ethers-core = { workspace = true, default-features = false, optional = true } -ethers-providers = { workspace = true, features = [ - "ws", -], default-features = false, optional = true } -ethers-middleware = { workspace = true, default-features = false, optional = true } -ethers-signers = { workspace = true, default-features = false, optional = true } - -# async / futures -async-trait = { workspace = true, optional = true } -tokio = { workspace = true, features = [ - "io-util", - "net", - "macros", - "rt-multi-thread", - "time", -], optional = true } - -# misc -hex = { version = "0.4", optional = true } - -[dev-dependencies] -# HACK(onbjerg): Workaround to enable test-utils when running tests -# Source: https://github.com/rust-lang/cargo/issues/2911#issuecomment-749580481 -reth-staged-sync = { path = ".", features = ["test-utils"] } - -# reth crates -reth-tracing = { path = "../tracing" } -reth-downloaders = { path = "../net/downloaders" } - -# async/futures -futures = { workspace = true } -tokio = { workspace = true, features = ["io-util", "net", "macros", "rt-multi-thread", "time"] } - -# crypto -secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] } - -confy = "0.5" - -tempfile = "3.4" -assert_matches = "1.5.0" - -[features] -test-utils = [ - "reth-network/test-utils", - "reth-network/test-utils", - "reth-provider/test-utils", - "dep:enr", - "dep:ethers-core", - "dep:hex", - "dep:rand", - "dep:tokio", - "dep:ethers-signers", - "dep:ethers-providers", - "dep:ethers-middleware", - "dep:async-trait", -] -geth-tests = [] diff --git a/crates/staged-sync/src/lib.rs b/crates/staged-sync/src/lib.rs deleted file mode 100644 index 7654d0bd393a..000000000000 --- a/crates/staged-sync/src/lib.rs +++ /dev/null @@ -1,24 +0,0 @@ -#![cfg_attr(docsrs, feature(doc_cfg))] -#![doc( - html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", - html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", - issue_tracker_base_url = "https://github.com/paradigmxzy/reth/issues/" -)] -#![warn(missing_docs, unreachable_pub)] -#![deny(unused_must_use, rust_2018_idioms)] -#![doc(test( - no_crate_inject, - attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) -))] - -//! Puts together all the Reth stages in a unified abstraction. -//! -//! ## Feature Flags -//! -//! - `test-utils`: Various utilities helpful for writing tests -//! - `geth-tests`: Runs tests that require Geth to be installed locally. -pub mod utils; - -#[cfg(any(test, feature = "test-utils"))] -/// Common helpers for integration testing. -pub mod test_utils; diff --git a/crates/staged-sync/src/utils/mod.rs b/crates/staged-sync/src/utils/mod.rs deleted file mode 100644 index 8c15ca28e8c3..000000000000 --- a/crates/staged-sync/src/utils/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Utility functions. - -/// Utilities for initializing parts of the chain -pub mod init; diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 01ec7bd0ff77..8418518fb4d2 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -86,7 +86,6 @@ These crates implement the main syncing drivers of reth. - [`blockchain-tree`](../../crates/blockchain-tree): A tree-like structure for handling multiple chains of unfinalized blocks. This is the main component during live sync (i.e. syncing at the tip) - [`stages`](../../crates/stages): A pipelined sync, including implementation of various stages. This is used during initial sync and is faster than the tree-like structure for longer sync ranges. -- [`staged-sync`](../../crates/staged-sync): A catch-all for various things currently, to be removed ### RPC