more clear randomness API for BABE (paritytech#8180)

* more clear randomness API for BABE

* babe: move randomness utilities to its own file

* node: use babe::RandomnessFromOneEpochAgo in random_seed implementation

* frame-support: annotate randomness trait with block number

* pallet-randomness-collective-flip: fix for new randomness trait

* pallet-society: fix randomness usage

* pallet-lottery: fix randomness usage

* pallet-contracts: fix randomness usage

* pallet-babe: fix randomness usage

we need to track when the current and previous epoch started so that we
know the block number by each existing on-chain was known

* node: fix random_seed

* node-template: fix random_seed

* frame-support: extend docs

* babe: add test for epoch starting block number tracking

* babe: fix epoch randomness docs

* frame: add todos for dealing with randomness api changes

Co-authored-by: André Silva <[email protected]>

bin/node-template/runtime/src/lib.rs

@@ -358,7 +358,7 @@ impl_runtime_apis! {
 		}
 
 		fn random_seed() -> <Block as BlockT>::Hash {
-			RandomnessCollectiveFlip::random_seed()
+			RandomnessCollectiveFlip::random_seed().0
 		}
 	}
 

bin/node/runtime/src/lib.rs

@@ -1220,7 +1220,7 @@ impl_runtime_apis! {
 		}
 
 		fn random_seed() -> <Block as BlockT>::Hash {
-			RandomnessCollectiveFlip::random_seed()
+			pallet_babe::RandomnessFromOneEpochAgo::<Runtime>::random_seed().0
 		}
 	}
 

frame/babe/src/lib.rs

@@ -25,15 +25,15 @@ use codec::{Decode, Encode};
 use frame_support::{
 	decl_error, decl_module, decl_storage,
 	dispatch::DispatchResultWithPostInfo,
-	traits::{FindAuthor, Get, KeyOwnerProofSystem, OneSessionHandler, Randomness as RandomnessT},
+	traits::{FindAuthor, Get, KeyOwnerProofSystem, OneSessionHandler},
 	weights::{Pays, Weight},
 	Parameter,
 };
 use frame_system::{ensure_none, ensure_root, ensure_signed};
 use sp_application_crypto::Public;
 use sp_runtime::{
 	generic::DigestItem,
-	traits::{Hash, IsMember, One, SaturatedConversion, Saturating, Zero},
+	traits::{IsMember, One, SaturatedConversion, Saturating, Zero},
 	ConsensusEngineId, KeyTypeId,
 };
 use sp_session::{GetSessionNumber, GetValidatorCount};
@@ -49,8 +49,9 @@ use sp_consensus_vrf::schnorrkel;
 
 pub use sp_consensus_babe::{AuthorityId, PUBLIC_KEY_LENGTH, RANDOMNESS_LENGTH, VRF_OUTPUT_LENGTH};
 
-mod equivocation;
 mod default_weights;
+mod equivocation;
+mod randomness;
 
 #[cfg(any(feature = "runtime-benchmarks", test))]
 mod benchmarking;
@@ -60,6 +61,9 @@ mod mock;
 mod tests;
 
 pub use equivocation::{BabeEquivocationOffence, EquivocationHandler, HandleEquivocation};
+pub use randomness::{
+	CurrentBlockRandomness, RandomnessFromOneEpochAgo, RandomnessFromTwoEpochsAgo,
+};
 
 pub trait Config: pallet_timestamp::Config {
 	/// The amount of time, in slots, that each epoch should last.
@@ -220,6 +224,13 @@ decl_storage! {
 		/// secondary plain slots are enabled (which don't contain a VRF output).
 		AuthorVrfRandomness get(fn author_vrf_randomness): MaybeRandomness;
 
+		/// The block numbers when the last and current epoch have started, respectively `N-1` and
+		/// `N`.
+		/// NOTE: We track this is in order to annotate the block number when a given pool of
+		/// entropy was fixed (i.e. it was known to chain observers). Since epochs are defined in
+		/// slots, which may be skipped, the block numbers may not line up with the slot numbers.
+		EpochStart: (T::BlockNumber, T::BlockNumber);
+
 		/// How late the current block is compared to its parent.
 		///
 		/// This entry is populated as part of block execution and is cleaned up
@@ -343,31 +354,6 @@ decl_module! {
 	}
 }
 
-impl<T: Config> RandomnessT<<T as frame_system::Config>::Hash> for Module<T> {
-	/// Some BABE blocks have VRF outputs where the block producer has exactly one bit of influence,
-	/// either they make the block or they do not make the block and thus someone else makes the
-	/// next block. Yet, this randomness is not fresh in all BABE blocks.
-	///
-	/// If that is an insufficient security guarantee then two things can be used to improve this
-	/// randomness:
-	///
-	/// - Name, in advance, the block number whose random value will be used; ensure your module
-	///   retains a buffer of previous random values for its subject and then index into these in
-	///   order to obviate the ability of your user to look up the parent hash and choose when to
-	///   transact based upon it.
-	/// - Require your user to first commit to an additional value by first posting its hash.
-	///   Require them to reveal the value to determine the final result, hashing it with the
-	///   output of this random function. This reduces the ability of a cabal of block producers
-	///   from conspiring against individuals.
-	fn random(subject: &[u8]) -> T::Hash {
-		let mut subject = subject.to_vec();
-		subject.reserve(VRF_OUTPUT_LENGTH);
-		subject.extend_from_slice(&Self::randomness()[..]);
-
-		<T as frame_system::Config>::Hashing::hash(&subject[..])
-	}
-}
-
 /// A BABE public key
 pub type BabeKey = [u8; PUBLIC_KEY_LENGTH];
 
@@ -492,6 +478,12 @@ impl<T: Config> Module<T> {
 		// Update the next epoch authorities.
 		NextAuthorities::put(&next_authorities);
 
+		// Update the start blocks of the previous and new current epoch.
+		<EpochStart<T>>::mutate(|(previous_epoch_start_block, current_epoch_start_block)| {
+			*previous_epoch_start_block = sp_std::mem::take(current_epoch_start_block);
+			*current_epoch_start_block = <frame_system::Module<T>>::block_number();
+		});
+
 		// After we update the current epoch, we signal the *next* epoch change
 		// so that nodes can track changes.
 		let next_randomness = NextRandomness::get();

frame/babe/src/randomness.rs

@@ -0,0 +1,148 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Provides multiple implementations of the randomness trait based on the on-chain epoch
+//! randomness collected from VRF outputs.
+
+use super::{
+	AuthorVrfRandomness, Config, EpochStart, NextRandomness, Randomness, VRF_OUTPUT_LENGTH,
+};
+use frame_support::{traits::Randomness as RandomnessT, StorageValue};
+use sp_runtime::traits::Hash;
+
+/// Randomness usable by consensus protocols that **depend** upon finality and take action
+/// based upon on-chain commitments made during the epoch before the previous epoch.
+///
+/// An off-chain consensus protocol requires randomness be finalized before usage, but one
+/// extra epoch delay beyond `RandomnessFromOneEpochAgo` suffices, under the assumption
+/// that finality never stalls for longer than one epoch.
+///
+/// All randomness is relative to commitments to any other inputs to the computation: If
+/// Alice samples randomness near perfectly using radioactive decay, but then afterwards
+/// Eve selects an arbitrary value with which to xor Alice's randomness, then Eve always
+/// wins whatever game they play.
+///
+/// All input commitments used with `RandomnessFromTwoEpochsAgo` should come from at least
+/// three epochs ago. We require BABE session keys be registered at least three epochs
+/// before being used to derive `CurrentBlockRandomness` for example.
+///
+/// All users learn `RandomnessFromTwoEpochsAgo` when epoch `current_epoch - 1` starts,
+/// although some learn it a few block earlier inside epoch `current_epoch - 2`.
+///
+/// Adversaries with enough block producers could bias this randomness by choosing upon
+/// what their block producers build at the end of epoch `current_epoch - 2` or the
+/// beginning epoch `current_epoch - 1`, or skipping slots at the end of epoch
+/// `current_epoch - 2`.
+///
+/// Adversaries should not possess many block production slots towards the beginning or
+/// end of every epoch, but they possess some influence over when they possess more slots.
+pub struct RandomnessFromTwoEpochsAgo<T>(sp_std::marker::PhantomData<T>);
+
+/// Randomness usable by on-chain code that **does not depend** upon finality and takes
+/// action based upon on-chain commitments made during the previous epoch.
+///
+/// All randomness is relative to commitments to any other inputs to the computation: If
+/// Alice samples randomness near perfectly using radioactive decay, but then afterwards
+/// Eve selects an arbitrary value with which to xor Alice's randomness, then Eve always
+/// wins whatever game they play.
+///
+/// All input commitments used with `RandomnessFromOneEpochAgo` should come from at least
+/// two epochs ago, although the previous epoch might work in special cases under
+/// additional assumption.
+///
+/// All users learn `RandomnessFromOneEpochAgo` at the end of the previous epoch, although
+/// some block producers learn it several block earlier.
+///
+/// Adversaries with enough block producers could bias this randomness by choosing upon
+/// what their block producers build at either the end of the previous epoch or the
+/// beginning of the current epoch, or electing to skipping some of their own block
+/// production slots towards the end of the previous epoch.
+///
+/// Adversaries should not possess many block production slots towards the beginning or
+/// end of every epoch, but they possess some influence over when they possess more slots.
+///
+/// As an example usage, we determine parachain auctions ending times in Polkadot using
+/// `RandomnessFromOneEpochAgo` because it reduces bias from `CurrentBlockRandomness` and
+/// does not require the extra finality delay of `RandomnessFromTwoEpochsAgo`.
+pub struct RandomnessFromOneEpochAgo<T>(sp_std::marker::PhantomData<T>);
+
+/// Randomness produced semi-freshly with each block, but inherits limitations of
+/// `RandomnessFromTwoEpochsAgo` from which it derives.
+///
+/// All randomness is relative to commitments to any other inputs to the computation: If
+/// Alice samples randomness near perfectly using radioactive decay, but then afterwards
+/// Eve selects an arbitrary value with which to xor Alice's randomness, then Eve always
+/// wins whatever game they play.
+///
+/// As with `RandomnessFromTwoEpochsAgo`, all input commitments combined with
+/// `CurrentBlockRandomness` should come from at least two epoch ago, except preferably
+/// not near epoch ending, and thus ideally three epochs ago.
+///
+/// Almost all users learn this randomness for a block when the block producer announces
+/// the block, which makes this randomness appear quite fresh. Yet, the block producer
+/// themselves learned this randomness at the beginning of epoch `current_epoch - 2`, at
+/// the same time as they learn `RandomnessFromTwoEpochsAgo`.
+///
+/// Aside from just biasing `RandomnessFromTwoEpochsAgo`, adversaries could also bias
+/// `CurrentBlockRandomness` by never announcing their block if doing so yields an
+/// unfavorable randomness. As such, `CurrentBlockRandomness` should be considered weaker
+/// than both other randomness sources provided by BABE, but `CurrentBlockRandomness`
+/// remains constrained by declared staking, while a randomness source like block hash is
+/// only constrained by adversaries' unknowable computational power.
+///
+/// As an example use, parachains could assign block production slots based upon the
+/// `CurrentBlockRandomness` of their relay parent or relay parent's parent, provided the
+/// parachain registers collators but avoids censorship sensitive functionality like
+/// slashing. Any parachain with slashing could operate BABE itself or perhaps better yet
+/// a BABE-like approach that derives its `CurrentBlockRandomness`, and authorizes block
+/// production, based upon the relay parent's `CurrentBlockRandomness` or more likely the
+/// relay parent's `RandomnessFromTwoEpochsAgo`.
+pub struct CurrentBlockRandomness<T>(sp_std::marker::PhantomData<T>);
+
+impl<T: Config> RandomnessT<T::Hash, T::BlockNumber> for RandomnessFromTwoEpochsAgo<T> {
+	fn random(subject: &[u8]) -> (T::Hash, T::BlockNumber) {
+		let mut subject = subject.to_vec();
+		subject.reserve(VRF_OUTPUT_LENGTH);
+		subject.extend_from_slice(&Randomness::get()[..]);
+
+		(T::Hashing::hash(&subject[..]), EpochStart::<T>::get().0)
+	}
+}
+
+impl<T: Config> RandomnessT<T::Hash, T::BlockNumber> for RandomnessFromOneEpochAgo<T> {
+	fn random(subject: &[u8]) -> (T::Hash, T::BlockNumber) {
+		let mut subject = subject.to_vec();
+		subject.reserve(VRF_OUTPUT_LENGTH);
+		subject.extend_from_slice(&NextRandomness::get()[..]);
+
+		(T::Hashing::hash(&subject[..]), EpochStart::<T>::get().1)
+	}
+}
+
+impl<T: Config> RandomnessT<Option<T::Hash>, T::BlockNumber> for CurrentBlockRandomness<T> {
+	fn random(subject: &[u8]) -> (Option<T::Hash>, T::BlockNumber) {
+		let random = AuthorVrfRandomness::get().map(|random| {
+			let mut subject = subject.to_vec();
+			subject.reserve(VRF_OUTPUT_LENGTH);
+			subject.extend_from_slice(&random);
+
+			T::Hashing::hash(&subject[..])
+		});
+
+		(random, <frame_system::Module<T>>::block_number())
+	}
+}

frame/babe/src/tests.rs

@@ -349,6 +349,31 @@ fn can_fetch_current_and_next_epoch_data() {
 	});
 }
 
+#[test]
+fn tracks_block_numbers_when_current_and_previous_epoch_started() {
+	new_test_ext(5).execute_with(|| {
+		// an epoch is 3 slots therefore at block 8 we should be in epoch #3
+		// with the previous epochs having the following blocks:
+		// epoch 1 - [1, 2, 3]
+		// epoch 2 - [4, 5, 6]
+		// epoch 3 - [7, 8, 9]
+		progress_to_block(8);
+
+		let (last_epoch, current_epoch) = EpochStart::<Test>::get();
+
+		assert_eq!(last_epoch, 4);
+		assert_eq!(current_epoch, 7);
+
+		// once we reach block 10 we switch to epoch #4
+		progress_to_block(10);
+
+		let (last_epoch, current_epoch) = EpochStart::<Test>::get();
+
+		assert_eq!(last_epoch, 7);
+		assert_eq!(current_epoch, 10);
+	});
+}
+
 #[test]
 fn report_equivocation_current_session_works() {
 	let (pairs, mut ext) = new_test_ext_with_pairs(3);

frame/contracts/src/exec.rs

@@ -740,7 +740,9 @@ where
 	}
 
 	fn random(&self, subject: &[u8]) -> SeedOf<T> {
-		T::Randomness::random(subject)
+		// TODO: change API to expose randomness freshness
+		// https://github.com/paritytech/substrate/issues/8297
+		T::Randomness::random(subject).0
 	}
 
 	fn now(&self) -> &MomentOf<T> {

frame/contracts/src/lib.rs

@@ -151,7 +151,7 @@ pub mod pallet {
 		type Time: Time;
 
 		/// The generator used to supply randomness to contracts through `seal_random`.
-		type Randomness: Randomness<Self::Hash>;
+		type Randomness: Randomness<Self::Hash, Self::BlockNumber>;
 
 		/// The currency in which fees are paid and contract balances are held.
 		type Currency: Currency<Self::AccountId>;

frame/lottery/Cargo.toml

@@ -22,6 +22,7 @@ frame-system = { version = "3.0.0", default-features = false, path = "../system"
 frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true }
 
 [dev-dependencies]
+frame-support-test = { version = "3.0.0", path = "../support/test" }
 pallet-balances = { version = "3.0.0", path = "../balances" }
 sp-core = { version = "3.0.0", path = "../../primitives/core" }
 sp-io = { version = "3.0.0", path = "../../primitives/io" }

frame/lottery/src/lib.rs

@@ -85,7 +85,7 @@ pub trait Config: frame_system::Config {
 	type Currency: ReservableCurrency<Self::AccountId>;
 
 	/// Something that provides randomness in the runtime.
-	type Randomness: Randomness<Self::Hash>;
+	type Randomness: Randomness<Self::Hash, Self::BlockNumber>;
 
 	/// The overarching event type.
 	type Event: From<Event<Self>> + Into<<Self as frame_system::Config>::Event>;
@@ -443,8 +443,10 @@ impl<T: Config> Module<T> {
 	// Note that there is potential bias introduced by using modulus operator.
 	// You should call this function with different seed values until the random
 	// number lies within `u32::MAX - u32::MAX % n`.
+	// TODO: deal with randomness freshness
+	// https://github.com/paritytech/substrate/issues/8311
 	fn generate_random_number(seed: u32) -> u32 {
-		let random_seed = T::Randomness::random(&(T::ModuleId::get(), seed).encode());
+		let (random_seed, _) = T::Randomness::random(&(T::ModuleId::get(), seed).encode());
 		let random_number = <u32>::decode(&mut random_seed.as_ref())
 			.expect("secure hashes should always be bigger than u32; qed");
 		random_number

frame/lottery/src/mock.rs

@@ -22,8 +22,9 @@ use crate as pallet_lottery;
 
 use frame_support::{
 	parameter_types,
-	traits::{OnInitialize, OnFinalize, TestRandomness},
+	traits::{OnFinalize, OnInitialize},
 };
+use frame_support_test::TestRandomness;
 use sp_core::H256;
 use sp_runtime::{
 	Perbill,
@@ -103,7 +104,7 @@ impl Config for Test {
 	type ModuleId = LotteryModuleId;
 	type Call = Call;
 	type Currency = Balances;
-	type Randomness = TestRandomness;
+	type Randomness = TestRandomness<Self>;
 	type Event = Event;
 	type ManagerOrigin = EnsureRoot<u64>;
 	type MaxCalls = MaxCalls;