Automatic project update

Author: tonjen

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 edition = '2018'
 name = 'ton_node'
-version = '0.48.9'
+version = '0.48.11'
 
 [workspace]
 members = [ 'storage' ]

commit_hash.txt

@@ -1 +1 @@
-eae0344e13e3a11918b3c830db28c68b24ccc77f
+9eb5cd04b1b913d5c07c7b01f863d66fdb835391

common

@@ -1 +1 @@
-{"adnl":"2587e9f5b9622940de20c51ef49e8d316e168b26","overlay":"864ea30a7fee351e6225d971a4d8836e3fec74c9","rldp":"14c7858004c3c941b97523cc49c1d83f47665eee","ton-labs-abi":"a74609a928d17cd02985ced2fcaf93f50321c21e","ton-tl":"8a4777b6bc1065019a9b5c52ac2a740a48fd926c","ton-block":"447a265d0013336b3c506cad45558922aa67f9f6","ton-block-json":"0fba14f552e67b9144404503612dc159b49d7e6b","ton-executor":"979410555a2961ffb080a112f8ebb4db90e466de","ton-types":"164867fa842b1efd0eeb378e727a17677ae8a322","ton-vm":"95512b5a06fb90695bfefda4ba99f7b0bd2d35d0"}
+{"adnl":"2587e9f5b9622940de20c51ef49e8d316e168b26","dht":"6529fcd71524c434407aa7240a1c89b2203e7b73","overlay":"864ea30a7fee351e6225d971a4d8836e3fec74c9","rldp":"14c7858004c3c941b97523cc49c1d83f47665eee","ton-labs-abi":"a74609a928d17cd02985ced2fcaf93f50321c21e","ton-tl":"8a4777b6bc1065019a9b5c52ac2a740a48fd926c","ton-block":"447a265d0013336b3c506cad45558922aa67f9f6","ton-block-json":"0fba14f552e67b9144404503612dc159b49d7e6b","ton-executor":"979410555a2961ffb080a112f8ebb4db90e466de","ton-types":"164867fa842b1efd0eeb378e727a17677ae8a322","ton-vm":"95512b5a06fb90695bfefda4ba99f7b0bd2d35d0"}

deps_map.json

@@ -37,10 +37,10 @@ use ton_block::{Block, BlockIdExt, AccountIdPrefixFull, UnixTime32};
 use ton_types::{error, fail, Result, UInt256};
 
 /// Full node state keys
-pub(crate) const INITIAL_MC_BLOCK: &str       = "InitMcBlockId";
-pub(crate) const LAST_APPLIED_MC_BLOCK: &str  = "LastMcBlockId";
-pub(crate) const PSS_KEEPER_MC_BLOCK: &str    = "PssKeeperBlockId";
-pub(crate) const SHARD_CLIENT_MC_BLOCK: &str  = "ShardsClientMcBlockId";
+pub const INITIAL_MC_BLOCK: &str       = "InitMcBlockId";
+pub const LAST_APPLIED_MC_BLOCK: &str  = "LastMcBlockId";
+pub const PSS_KEEPER_MC_BLOCK: &str    = "PssKeeperBlockId";
+pub const SHARD_CLIENT_MC_BLOCK: &str  = "ShardsClientMcBlockId";
 
 /// Validator state keys
 pub(crate) const LAST_ROTATION_MC_BLOCK: &str = "LastRotationBlockId";

src/internal_db/mod.rs

@@ -837,7 +837,7 @@ impl Collator {
             mc_data.mc_state_extra().shards(),
             mc_data.config(),
             now,
-            false,
+            self.collator_settings.is_fake
         )?;
 
         self.check_utime(&mc_data, &prev_data, &mut collator_data)?;

src/validator/collator_sync.rs

@@ -50,6 +50,17 @@ pub struct CollatorSettings {
     pub want_split: Option<bool>,
     pub want_merge: Option<bool>,
     pub max_collate_threads: Option<usize>,
+    pub is_fake: bool,
+}
+
+impl CollatorSettings {
+    #[allow(dead_code)]
+    pub fn fake() -> Self {
+        Self {
+            is_fake: true,
+            ..Self::default()
+        }
+    }
 }
 
 pub struct McData {

src/validator/mod.rs

@@ -27,42 +27,42 @@ use crate::db::rocksdb::RocksDb;
 use std::{
     io::{Cursor, Read, Write}, ops::Deref, 
     sync::{Arc, atomic::{AtomicU8, AtomicU32, Ordering}}, 
-    time::{Duration, Instant, SystemTime, UNIX_EPOCH}
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 use ton_block::BlockIdExt;
 use ton_types::{ByteOrderRead, Cell, Result, fail};
 
 pub(crate) struct DbEntry {
     pub cell_id: CellId, // TODO: remove this id
-    pub block_id_ext: BlockIdExt,
+    pub block_id: BlockIdExt,
     pub db_index: u32,
-    pub saved_at: u64,
+    pub save_utime: u64,
 }
 
 impl DbEntry {
-    pub fn with_params(cell_id: CellId, block_id_ext: BlockIdExt, db_index: u32, saved_at: u64) -> Self {
-        Self { cell_id, block_id_ext, db_index, saved_at }
+    pub fn with_params(cell_id: CellId, block_id: BlockIdExt, db_index: u32, save_utime: u64) -> Self {
+        Self { cell_id, block_id, db_index, save_utime }
     }
 }
 
 impl Serializable for DbEntry {
     fn serialize<T: Write>(&self, writer: &mut T) -> Result<()> {
         writer.write_all(self.cell_id.key())?;
-        self.block_id_ext.serialize(writer)?;
+        self.block_id.serialize(writer)?;
         writer.write_all(&self.db_index.to_le_bytes())?;
-        writer.write_all(&self.saved_at.to_le_bytes())?;
+        writer.write_all(&self.save_utime.to_le_bytes())?;
         Ok(())
     }
 
     fn deserialize<T: Read>(reader: &mut T) -> Result<Self> {
         let mut buf = [0; 32];
         reader.read_exact(&mut buf)?;
         let cell_id = CellId::new(buf.into());
-        let block_id_ext = BlockIdExt::deserialize(reader)?;
+        let block_id = BlockIdExt::deserialize(reader)?;
         let db_index = reader.read_le_u32().unwrap_or(0); // use 0 db by default
-        let saved_at = reader.read_le_u64().unwrap_or(0);
+        let save_utime = reader.read_le_u64().unwrap_or(0);
 
-        Ok(Self { cell_id, block_id_ext, db_index, saved_at })
+        Ok(Self { cell_id, block_id, db_index, save_utime })
     }
 }
 
@@ -76,6 +76,8 @@ pub struct ShardStateDb {
     stop: AtomicU8
 }
 
+const SHARDSTATES_GC_LAG_SEC: u64 = 300;
+
 impl ShardStateDb {
 
     const MASK_GC_STARTED: u8 = 0x01;
@@ -305,8 +307,8 @@ impl ShardStateDb {
 
         let c = boc_db.save_as_dynamic_boc(state_root.deref())?;
 
-        let saved_at = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
-        let db_entry = DbEntry::with_params(cell_id.clone(), id.clone(), boc_db.db_index(), saved_at);
+        let save_utime = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
+        let db_entry = DbEntry::with_params(cell_id.clone(), id.clone(), boc_db.db_index(), save_utime);
         let mut buf = Vec::new();
         db_entry.serialize(&mut Cursor::new(&mut buf))?;
 
@@ -341,7 +343,7 @@ impl ShardStateDb {
 }
 
 pub trait AllowStateGcResolver: Send + Sync {
-    fn allow_state_gc(&self, block_id: &BlockIdExt, saved_at: u64, gc_utime: u64) -> Result<bool>;
+    fn allow_state_gc(&self, block_id: &BlockIdExt, save_utime: u64, gc_utime: u64) -> Result<bool>;
 }
 
 #[derive(Default)]
@@ -416,26 +418,51 @@ fn mark(
     gc_resolver: &dyn AllowStateGcResolver,
     gc_time: u64
 ) -> MarkResult {
-    let mut to_mark = Vec::new();
-    let mut to_sweep = Vec::new();
+    // We should leave last states in DB to avoid full state saving when DB will active.
+    // 1) enumerate all roots with their save-time
+    // 2) remember max time and max masterchain root
+    // 3) do not sweep last masterchain root and shardchains roots 
+    //    for last SHARDSTATES_GC_LAG_SEC seconds
+
+    let mut last_mc_seqno = 0;
+    let mut roots = Vec::new();
+    let mut max_shard_utime = 0;
+
     shardstate_db.for_each(&mut |_key, value| {
         let db_entry = DbEntry::from_slice(value)?;
-        let cell_id = db_entry.cell_id;
-        let block_id_ext = db_entry.block_id_ext;
         if db_entry.db_index == dynamic_boc_db.db_index() {
-            if gc_resolver.allow_state_gc(&block_id_ext, db_entry.saved_at, gc_time)? {
-                log::trace!(target: TARGET, "mark  to_sweep  block_id {}  cell_id {}  db_index {} ", 
-                    block_id_ext, cell_id, dynamic_boc_db.db_index());
-                to_sweep.push((block_id_ext, cell_id));
+            if db_entry.block_id.shard().is_masterchain() {
+                if db_entry.block_id.seq_no() > last_mc_seqno {
+                    last_mc_seqno = db_entry.block_id.seq_no();
+                }
             } else {
-                log::trace!(target: TARGET, "mark  to_mark  block_id {}  cell_id {}  db_index {} ", 
-                    block_id_ext, cell_id, dynamic_boc_db.db_index());
-                to_mark.push(cell_id);
+                if db_entry.save_utime > max_shard_utime{
+                    max_shard_utime = db_entry.save_utime;
+                }
             }
+            roots.push(db_entry);
         }
         Ok(true)
     })?;
 
+    let mut to_mark = Vec::new();
+    let mut to_sweep = Vec::new();
+    for root in roots {
+        let is_mc = root.block_id.shard().is_masterchain();
+        if gc_resolver.allow_state_gc(&root.block_id, root.save_utime, gc_time)?
+            && (!is_mc || root.block_id.seq_no() < last_mc_seqno)
+            && (is_mc || root.save_utime + SHARDSTATES_GC_LAG_SEC < max_shard_utime)
+        {
+            log::trace!(target: TARGET, "mark  to_sweep  block_id {}  cell_id {}  db_index {}  utime {}", 
+                root.block_id, root.cell_id, dynamic_boc_db.db_index(), root.save_utime);
+            to_sweep.push((root.block_id, root.cell_id));
+        } else {
+            log::trace!(target: TARGET, "mark  to_mark  block_id {}  cell_id {}  db_index {}  utime {}", 
+                root.block_id, root.cell_id, dynamic_boc_db.db_index(), root.save_utime);
+            to_mark.push(root.cell_id);
+        }
+    }
+
     let marked_roots = to_mark.len();
     let mut marked = FnvHashSet::default();
     if !to_sweep.is_empty() {