Code, PoV compression and remove CompressedPoV struct (paritytech#2852

)

* use compressed blob in candidate-validation

* add some tests for compressed code blobs

* remove CompressedPoV and apply compression in collation-generation

* decompress BlockData before executing

* don't produce oversized collations

* add test for PoV decompression failure

* fix tests and clean up

* fix test

* address review and fix CI

* take this )

node/collation-generation/Cargo.toml

@@ -13,7 +13,9 @@ polkadot-node-subsystem = { path = "../subsystem" }
 polkadot-node-subsystem-util = { path = "../subsystem-util" }
 polkadot-primitives = { path = "../../primitives" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob  = { git = "https://github.com/paritytech/substrate", branch = "master" }
 thiserror = "1.0.23"
+parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] }
 
 [dev-dependencies]
 polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }

node/collation-generation/src/lib.rs

@@ -26,7 +26,9 @@ use futures::{
 	sink::SinkExt,
 	stream::StreamExt,
 };
-use polkadot_node_primitives::{CollationGenerationConfig, AvailableData, PoV};
+use polkadot_node_primitives::{
+	CollationGenerationConfig, AvailableData, PoV,
+};
 use polkadot_node_subsystem::{
 	messages::{AllMessages, CollationGenerationMessage, CollatorProtocolMessage},
 	FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemResult,
@@ -41,6 +43,7 @@ use polkadot_primitives::v1::{
 	CandidateDescriptor, CandidateReceipt, CoreState, Hash, OccupiedCoreAssumption,
 	PersistedValidationData,
 };
+use parity_scale_codec::Encode;
 use sp_core::crypto::Pair;
 use std::sync::Arc;
 
@@ -313,7 +316,32 @@ async fn handle_new_activations<Context: SubsystemContext>(
 					}
 				};
 
-				let pov_hash = collation.proof_of_validity.hash();
+				// Apply compression to the block data.
+				let pov = {
+					let pov = polkadot_node_primitives::maybe_compress_pov(collation.proof_of_validity);
+					let encoded_size = pov.encoded_size();
+
+					// As long as `POV_BOMB_LIMIT` is at least `max_pov_size`, this ensures
+					// that honest collators never produce a PoV which is uncompressed.
+					//
+					// As such, honest collators never produce an uncompressed PoV which starts with
+					// a compression magic number, which would lead validators to reject the collation.
+					if encoded_size > validation_data.max_pov_size as usize {
+						tracing::debug!(
+							target: LOG_TARGET,
+							para_id = %scheduled_core.para_id,
+							size = encoded_size,
+							max_size = validation_data.max_pov_size,
+							"PoV exceeded maximum size"
+						);
+
+						return
+					}
+
+					pov
+				};
+
+				let pov_hash = pov.hash();
 
 				let signature_payload = collator_signature_payload(
 					&relay_parent,
@@ -326,7 +354,7 @@ async fn handle_new_activations<Context: SubsystemContext>(
 				let erasure_root = match erasure_root(
 					n_validators,
 					validation_data,
-					collation.proof_of_validity.clone(),
+					pov.clone(),
 				) {
 					Ok(erasure_root) => erasure_root,
 					Err(err) => {
@@ -375,7 +403,7 @@ async fn handle_new_activations<Context: SubsystemContext>(
 				metrics.on_collation_generated();
 
 				if let Err(err) = task_sender.send(AllMessages::CollatorProtocol(
-					CollatorProtocolMessage::DistributeCollation(ccr, collation.proof_of_validity, result_sender)
+					CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender)
 				)).await {
 					tracing::warn!(
 						target: LOG_TARGET,
@@ -492,16 +520,15 @@ mod tests {
 			task::{Context as FuturesContext, Poll},
 			Future,
 		};
-		use polkadot_node_primitives::{Collation, CollationResult, BlockData, PoV};
+		use polkadot_node_primitives::{Collation, CollationResult, BlockData, PoV, POV_BOMB_LIMIT};
 		use polkadot_node_subsystem::messages::{
 			AllMessages, RuntimeApiMessage, RuntimeApiRequest,
 		};
 		use polkadot_node_subsystem_test_helpers::{
 			subsystem_test_harness, TestSubsystemContextHandle,
 		};
 		use polkadot_primitives::v1::{
-			BlockNumber, CollatorPair, Id as ParaId,
-			PersistedValidationData, ScheduledCore, ValidationCode,
+			CollatorPair, Id as ParaId, PersistedValidationData, ScheduledCore, ValidationCode,
 		};
 		use std::pin::Pin;
 
@@ -519,6 +546,24 @@ mod tests {
 			}
 		}
 
+		fn test_collation_compressed() -> Collation {
+			let mut collation = test_collation();
+			let compressed = PoV {
+				block_data: BlockData(sp_maybe_compressed_blob::compress(
+					&collation.proof_of_validity.block_data.0,
+					POV_BOMB_LIMIT,
+				).unwrap())
+			};
+			collation.proof_of_validity = compressed;
+			collation
+		}
+
+		fn test_validation_data() -> PersistedValidationData {
+			let mut persisted_validation_data: PersistedValidationData = Default::default();
+			persisted_validation_data.max_pov_size = 1024;
+			persisted_validation_data
+		}
+
 		// Box<dyn Future<Output = Collation> + Unpin + Send
 		struct TestCollator;
 
@@ -715,7 +760,7 @@ mod tests {
 								tx,
 							),
 						))) => {
-							tx.send(Ok(Some(Default::default()))).unwrap();
+							tx.send(Ok(Some(test_validation_data()))).unwrap();
 						}
 						Some(AllMessages::RuntimeApi(RuntimeApiMessage::Request(
 							_hash,
@@ -766,9 +811,8 @@ mod tests {
 			// we expect a single message to be sent, containing a candidate receipt.
 			// we don't care too much about the commitments_hash right now, but let's ensure that we've calculated the
 			// correct descriptor
-			let expect_pov_hash = test_collation().proof_of_validity.hash();
-			let expect_validation_data_hash
-				= PersistedValidationData::<Hash, BlockNumber>::default().hash();
+			let expect_pov_hash = test_collation_compressed().proof_of_validity.hash();
+			let expect_validation_data_hash = test_validation_data().hash();
 			let expect_relay_parent = Hash::repeat_byte(4);
 			let expect_validation_code_hash = ValidationCode(vec![1, 2, 3]).hash();
 			let expect_payload = collator_signature_payload(

node/core/candidate-validation/Cargo.toml

@@ -9,6 +9,7 @@ futures = "0.3.12"
 tracing = "0.1.25"
 
 sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob = { package = "sp-maybe-compressed-blob", git = "https://github.com/paritytech/substrate", branch = "master" }
 parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] }
 
 polkadot-primitives = { path = "../../../primitives" }