Skip to content

Commit

Permalink
import-blocks: Do not read stdin to memory (paritytech#11072)
Browse files Browse the repository at this point in the history
* import-blocks: Do not read `stdin` to memory

This fixes a bug with `import-blocks` reading the entire `stdin` before starting to import the
blocks. However, for huge files that uses quite a lot of memory. We can just read from `stdin` step
by step as we do it with a file. This ensures that we don't read the entire input at once into memory.

* FMT

* Fix warning
  • Loading branch information
bkchr authored Mar 21, 2022
1 parent 5917c0e commit 5d630dd
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 13 deletions.
8 changes: 2 additions & 6 deletions client/cli/src/commands/import_blocks_cmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,9 @@ impl ImportBlocksCmd {
B: BlockT + for<'de> serde::Deserialize<'de>,
IQ: sc_service::ImportQueue<B> + 'static,
{
let file: Box<dyn ReadPlusSeek + Send> = match &self.input {
let file: Box<dyn Read + Send> = match &self.input {
Some(filename) => Box::new(fs::File::open(filename)?),
None => {
let mut buffer = Vec::new();
io::stdin().read_to_end(&mut buffer)?;
Box::new(io::Cursor::new(buffer))
},
None => Box::new(io::stdin()),
};

import_blocks(client, import_queue, file, false, self.binary)
Expand Down
15 changes: 8 additions & 7 deletions client/service/src/chain_ops/import_blocks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ use sp_runtime::{
};
use std::{
convert::{TryFrom, TryInto},
io::{Read, Seek},
io::Read,
pin::Pin,
task::Poll,
time::{Duration, Instant},
Expand All @@ -63,7 +63,7 @@ pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result<String> {
/// SignedBlock and return it.
enum BlockIter<R, B>
where
R: std::io::Read + std::io::Seek,
R: std::io::Read,
{
Binary {
// Total number of blocks we are expecting to decode.
Expand All @@ -83,7 +83,7 @@ where

impl<R, B> BlockIter<R, B>
where
R: Read + Seek + 'static,
R: Read + 'static,
B: BlockT + MaybeSerializeDeserialize,
{
fn new(input: R, binary: bool) -> Result<Self, String> {
Expand Down Expand Up @@ -119,7 +119,7 @@ where

impl<R, B> Iterator for BlockIter<R, B>
where
R: Read + Seek + 'static,
R: Read + 'static,
B: BlockT + MaybeSerializeDeserialize,
{
type Item = Result<SignedBlock<B>, String>;
Expand Down Expand Up @@ -267,10 +267,11 @@ impl<B: BlockT> Speedometer<B> {
/// Different State that the `import_blocks` future could be in.
enum ImportState<R, B>
where
R: Read + Seek + 'static,
R: Read + 'static,
B: BlockT + MaybeSerializeDeserialize,
{
/// We are reading from the BlockIter structure, adding those blocks to the queue if possible.
/// We are reading from the [`BlockIter`] structure, adding those blocks to the queue if
/// possible.
Reading { block_iter: BlockIter<R, B> },
/// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it
/// to catch up.
Expand All @@ -291,7 +292,7 @@ where
pub fn import_blocks<B, IQ, C>(
client: Arc<C>,
mut import_queue: IQ,
input: impl Read + Seek + Send + 'static,
input: impl Read + Send + 'static,
force: bool,
binary: bool,
) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>
Expand Down

0 comments on commit 5d630dd

Please sign in to comment.