Storage layer (substrate + interfaces) (near#72)

* Added Authority tracking (for now just initial authority). Added Signer trait that can sign hash of data and InMemorySigner implementation

* Added staking to the runtime

* Add AuthorityChangeSet that Runtime outputs for further consumption of Authority class

* Adding PR comments

* Added Authority tracking (for now just initial authority). Added Signer trait that can sign hash of data and InMemorySigner implementation

* Add AuthorityChangeSet that Runtime outputs for further consumption of Authority class

* Together with @mikhailOK worked on integrating substrate state into our storage layer

* Cleaning up the usage of Storage, enabling tests in Runtime and Client to work; Rust formatting added.

* @bowenwang1996: ganache with rpc server and block producing task

* Block production includes blocks into blockchain; Fixup tests to not use default transactions which are incorrect

.gitignore

@@ -2,8 +2,8 @@
 # will have compiled files and executables
 **/target/
 /storage/
-/node/service/storage
 /test-utils/node/storage
+/test-utils/ganache/storage
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html

Cargo.toml

@@ -24,7 +24,8 @@ members = [
     "node/runtime",
     "node/service",
     "node/network",
-    "test-utils/node"
+    "test-utils/node",
+    "test-utils/ganache"
 ]
 exclude = [
     "core/wasm/runtest/generate-wasm/to-wasm",

core/beacon/src/authority.rs

@@ -0,0 +1,87 @@
+use chain::BlockChain;
+use primitives::signature::PublicKey;
+use primitives::types::AccountId;
+use std::collections::HashMap;
+use types::BeaconBlock;
+
+/// Configure the authority rotation.
+pub struct AuthorityConfig {
+    /// List of initial authorities at genesis block.
+    pub initial_authorities: Vec<PublicKey>,
+    /// Authority epoch length.
+    pub epoch_length: u64,
+}
+
+#[derive(Default)]
+pub struct AuthorityChangeSet {
+    pub proposed: HashMap<AccountId, (PublicKey, u64)>,
+}
+
+pub struct Authority {
+    /// Authority configuation.
+    authority_config: AuthorityConfig,
+    /// Cache of current authorities for given index.
+    _current: HashMap<u64, Vec<PublicKey>>,
+}
+
+impl Authority {
+    /// Builds authority for given valid blockchain.
+    /// Starting from best block, figure out current authorities.
+    pub fn new(authority_config: AuthorityConfig, blockchain: &BlockChain<BeaconBlock>) -> Self {
+        let authority = Authority { authority_config, _current: HashMap::default() };
+        let last_index = blockchain.best_block().header.index;
+        let _current_epoch = last_index / authority.authority_config.epoch_length;
+
+        authority
+    }
+
+    /// Returns authorities for given block number.
+    pub fn get_authorities(&self, _index: u64) -> Vec<PublicKey> {
+        // TODO: use cache to retrieve current authorities or compute future ones.
+        self.authority_config.initial_authorities.clone()
+    }
+
+    /// Updates authority for given block.
+    pub fn update_authority(&mut self, _change_set: &AuthorityChangeSet) {
+        // TODO: update cache of current authorities.
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    use chain::ChainConfig;
+    use primitives::hash::CryptoHash;
+    use primitives::traits::Block;
+    use primitives::types::MerkleHash;
+    use std::sync::Arc;
+    use storage::test_utils::MemoryStorage;
+
+    fn test_blockchain(num_blocks: u64) -> BlockChain<BeaconBlock> {
+        let storage = Arc::new(MemoryStorage::default());
+        let chain_config = ChainConfig {
+            extra_col: storage::COL_BEACON_EXTRA,
+            header_col: storage::COL_BEACON_HEADERS,
+            block_col: storage::COL_BEACON_BLOCKS,
+            index_col: storage::COL_BEACON_INDEX,
+        };
+        let mut last_block =
+            BeaconBlock::new(0, CryptoHash::default(), MerkleHash::default(), vec![]);
+        let bc = BlockChain::new(chain_config, last_block.clone(), storage);
+        for i in 1..num_blocks {
+            let block = BeaconBlock::new(i, last_block.hash(), MerkleHash::default(), vec![]);
+            bc.insert_block(block.clone());
+            last_block = block;
+        }
+        bc
+    }
+
+    #[test]
+    fn test_authority_genesis() {
+        let authority_config = AuthorityConfig { initial_authorities: vec![], epoch_length: 2 };
+        let bc = test_blockchain(0);
+        let authority = Authority::new(authority_config, &bc);
+        assert_eq!(authority.get_authorities(0), vec![]);
+    }
+}

core/beacon/src/chain.rs

@@ -13,13 +13,13 @@ const BLOCKCHAIN_BEST_BLOCK: &[u8] = b"best";
 #[derive(Clone)]
 pub struct ChainConfig {
     /// Column in storage for extra information.
-    pub extra_col: u32,
+    pub extra_col: Option<u32>,
     /// Column in storage for blocks.
-    pub block_col: u32,
+    pub block_col: Option<u32>,
     /// Column in storage for headers.
-    pub header_col: u32,
+    pub header_col: Option<u32>,
     /// Column in storage for index.
-    pub index_col: u32,
+    pub index_col: Option<u32>,
 }
 
 /// General BlockChain container.
@@ -43,27 +43,28 @@ pub struct BlockChain<B: Block> {
 
 fn index_to_bytes(index: u64) -> Vec<u8> {
     let mut bytes = vec![];
-    bytes
-        .write_u64::<LittleEndian>(index)
-        .expect("writing to bytes failed");
+    bytes.write_u64::<LittleEndian>(index).expect("writing to bytes failed");
     bytes
 }
 
 fn write_with_cache<T: Clone + Encode>(
     storage: &Arc<Storage>,
-    col: u32,
+    col: Option<u32>,
     cache: &RwLock<HashMap<Vec<u8>, T>>,
     key: &[u8],
     value: &T,
 ) {
     let data = Encode::encode(value).expect("Error serializing data");
     cache.write().insert(key.to_vec(), value.clone());
-    storage.set(col, key, &data);
+
+    let mut db_transaction = storage.transaction();
+    db_transaction.put(col, key, &data);
+    storage.write(db_transaction).expect("Database write failed");
 }
 
 fn read_with_cache<T: Clone + Decode>(
     storage: &Arc<Storage>,
-    col: u32,
+    col: Option<u32>,
     cache: &RwLock<HashMap<Vec<u8>, T>>,
     key: &[u8],
 ) -> Option<T> {
@@ -74,12 +75,14 @@ fn read_with_cache<T: Clone + Decode>(
         }
     }
 
-    // TODO: use columns here.
-    Decode::decode(&storage.get(col, key)?).map(|value: T| {
-        let mut write = cache.write();
-        write.insert(key.to_vec(), value.clone());
-        value
-    })
+    match storage.get(col, key) {
+        Ok(Some(value)) => Decode::decode(value.as_ref()).map(|value: T| {
+            let mut write = cache.write();
+            write.insert(key.to_vec(), value.clone());
+            value
+        }),
+        _ => None,
+    }
 }
 
 impl<B: Block> BlockChain<B> {
@@ -96,11 +99,9 @@ impl<B: Block> BlockChain<B> {
         };
 
         // Load best block hash from storage.
-        let best_block_hash = match bc
-            .storage
-            .get(bc.chain_config.extra_col, BLOCKCHAIN_BEST_BLOCK)
+        let best_block_hash = match bc.storage.get(bc.chain_config.extra_col, BLOCKCHAIN_BEST_BLOCK)
         {
-            Some(best_hash) => CryptoHash::new(&best_hash),
+            Ok(Some(best_hash)) => CryptoHash::new(best_hash.as_ref()),
             _ => {
                 // Insert genesis block into cache.
                 bc.insert_block(genesis);
@@ -125,9 +126,13 @@ impl<B: Block> BlockChain<B> {
 
     /// Check if block already is known.
     pub fn is_known(&self, hash: &CryptoHash) -> bool {
-        self.headers.read().contains_key(hash.as_ref()) || self
-            .storage
-            .exists(self.chain_config.header_col, hash.as_ref())
+        if self.headers.read().contains_key(hash.as_ref()) {
+            return true;
+        }
+        match self.storage.get(self.chain_config.header_col, hash.as_ref()) {
+            Ok(Some(_)) => true,
+            _ => false,
+        }
     }
 
     /// Inserts a verified block.
@@ -167,11 +172,13 @@ impl<B: Block> BlockChain<B> {
             if block.header().index() > self.best_block.read().header().index() {
                 let mut best_block = self.best_block.write();
                 *best_block = block;
-                self.storage.set(
+                let mut db_transaction = self.storage.transaction();
+                db_transaction.put(
                     self.chain_config.extra_col,
                     BLOCKCHAIN_BEST_BLOCK,
                     block_hash.as_ref(),
                 );
+                self.storage.write(db_transaction).expect("Database write failed");
             }
             false
         } else {
@@ -227,51 +234,45 @@ impl<B: Block> BlockChain<B> {
 mod tests {
     use super::*;
 
-    use primitives::types::BLSSignature;
+    use primitives::types::MerkleHash;
     use std::sync::Arc;
-    use storage::MemoryStorage;
+    use storage::test_utils::create_memory_db;
     use types::BeaconBlock;
 
     #[test]
     fn test_genesis() {
-        let storage = Arc::new(MemoryStorage::default());
+        let storage = Arc::new(create_memory_db());
         let chain_config = ChainConfig {
-            extra_col: 0,
-            header_col: 1,
-            block_col: 2,
-            index_col: 3,
+            extra_col: storage::COL_BEACON_EXTRA,
+            header_col: storage::COL_BEACON_HEADERS,
+            block_col: storage::COL_BEACON_BLOCKS,
+            index_col: storage::COL_BEACON_INDEX,
         };
-        let genesis = BeaconBlock::new(0, CryptoHash::default(), BLSSignature::default(), vec![]);
+        let genesis = BeaconBlock::new(0, CryptoHash::default(), MerkleHash::default(), vec![]);
         let bc = BlockChain::new(chain_config, genesis.clone(), storage);
-        assert_eq!(
-            bc.get_block(&BlockId::Hash(genesis.hash())).unwrap(),
-            genesis
-        );
+        assert_eq!(bc.get_block(&BlockId::Hash(genesis.hash())).unwrap(), genesis);
         assert_eq!(bc.get_block(&BlockId::Number(0)).unwrap(), genesis);
     }
 
     #[test]
     fn test_restart_chain() {
-        let storage = Arc::new(MemoryStorage::default());
+        let storage = Arc::new(create_memory_db());
         let chain_config = ChainConfig {
-            extra_col: 0,
-            header_col: 1,
-            block_col: 2,
-            index_col: 3,
+            extra_col: storage::COL_BEACON_EXTRA,
+            header_col: storage::COL_BEACON_HEADERS,
+            block_col: storage::COL_BEACON_BLOCKS,
+            index_col: storage::COL_BEACON_INDEX,
         };
-        let genesis = BeaconBlock::new(0, CryptoHash::default(), BLSSignature::default(), vec![]);
+        let genesis = BeaconBlock::new(0, CryptoHash::default(), MerkleHash::default(), vec![]);
         let bc = BlockChain::new(chain_config.clone(), genesis.clone(), storage.clone());
-        let block1 = BeaconBlock::new(1, genesis.hash(), BLSSignature::default(), vec![]);
+        let block1 = BeaconBlock::new(1, genesis.hash(), MerkleHash::default(), vec![]);
         assert_eq!(bc.insert_block(block1.clone()), false);
         assert_eq!(bc.best_block().hash(), block1.hash());
         assert_eq!(bc.best_block().header.index, 1);
         // Create new BlockChain that reads from the same storage.
         let other_bc = BlockChain::new(chain_config, genesis.clone(), storage.clone());
         assert_eq!(other_bc.best_block().hash(), block1.hash());
         assert_eq!(other_bc.best_block().header.index, 1);
-        assert_eq!(
-            other_bc.get_block(&BlockId::Hash(block1.hash())).unwrap(),
-            block1
-        );
+        assert_eq!(other_bc.get_block(&BlockId::Hash(block1.hash())).unwrap(), block1);
     }
 }

core/beacon/src/lib.rs

@@ -6,5 +6,6 @@ extern crate byteorder;
 extern crate primitives;
 extern crate storage;
 
+pub mod authority;
 pub mod chain;
 pub mod types;

core/beacon/src/types.rs

@@ -1,8 +1,9 @@
 use primitives::hash::{hash_struct, CryptoHash};
-use primitives::traits::{Block, Header};
+use primitives::traits::{Block, Header, Signer};
 use primitives::types::{BLSSignature, MerkleHash, SignedTransaction};
+use std::sync::Arc;
 
-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Default)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct BeaconBlockHeader {
     /// Parent hash.
     pub parent_hash: CryptoHash,
@@ -39,20 +40,24 @@ impl BeaconBlock {
     pub fn new(
         index: u64,
         parent_hash: CryptoHash,
-        signature: BLSSignature,
+        merkle_root_state: MerkleHash,
         transactions: Vec<SignedTransaction>,
     ) -> Self {
         BeaconBlock {
             header: BeaconBlockHeader {
                 parent_hash,
                 merkle_root_tx: MerkleHash::default(),
-                merkle_root_state: MerkleHash::default(),
-                signature,
+                merkle_root_state,
+                signature: primitives::signature::default_signature(),
                 index,
             },
             transactions,
         }
     }
+
+    pub fn sign(&mut self, signer: &Arc<Signer>) {
+        self.header.signature = signer.sign(&self.hash());
+    }
 }
 
 impl Block for BeaconBlock {
@@ -72,10 +77,7 @@ impl Block for BeaconBlock {
     }
 
     fn new(header: Self::Header, body: Self::Body) -> Self {
-        BeaconBlock {
-            header,
-            transactions: body,
-        }
+        BeaconBlock { header, transactions: body }
     }
 
     fn hash(&self) -> CryptoHash {

core/primitives/Cargo.toml

@@ -9,4 +9,5 @@ serde = "1.0"
 serde_derive = "1.0"
 sha2 = "0.7.0"
 exonum_sodiumoxide = "0.0.20"
-bincode = "1.0.0"
+bincode = "1.0.0"
+heapsize = "0.4"

core/primitives/src/hash.rs

@@ -50,3 +50,9 @@ pub fn hash(data: &[u8]) -> CryptoHash {
 pub fn hash_struct<T: Encode>(obj: &T) -> CryptoHash {
     hash(&obj.encode().expect("Serialization failed"))
 }
+
+impl heapsize::HeapSizeOf for CryptoHash {
+    fn heap_size_of_children(&self) -> usize {
+        0
+    }
+}

core/primitives/src/lib.rs

@@ -1,10 +1,12 @@
+extern crate bincode;
+extern crate exonum_sodiumoxide;
+extern crate heapsize;
 extern crate serde;
 #[macro_use]
 extern crate serde_derive;
-extern crate bincode;
-extern crate exonum_sodiumoxide;
 
 pub mod hash;
 pub mod signature;
+pub mod signer;
 pub mod traits;
 pub mod types;

core/primitives/src/signature.rs

@@ -17,3 +17,8 @@ pub fn get_keypair_from_seed(seed: &Seed) -> (PublicKey, SecretKey) {
 pub fn get_keypair() -> (PublicKey, SecretKey) {
     sodiumoxide::crypto::sign::ed25519::gen_keypair()
 }
+
+pub fn default_signature() -> Signature {
+    let sig = [0u8; sodiumoxide::crypto::sign::ed25519::SIGNATUREBYTES];
+    sodiumoxide::crypto::sign::ed25519::Signature(sig)
+}