Skip to content

Commit

Permalink
More benchmark machine args (paritytech#11428)
Browse files Browse the repository at this point in the history 
* Add ExecutionLimits to sc-sysinfo and return float

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Increase benchmarking duration and add options

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <[email protected]>
  • Loading branch information
@ggwpez
ggwpez authored May 17, 2022
1 parent 17e062b commit c44cd63
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 29 deletions.
24 changes: 22 additions & 2 deletions bin/node/cli/tests/benchmark_machine_works.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,16 @@ use std::process::Command;
fn benchmark_machine_works() {
let status = Command::new(cargo_bin("substrate"))
.args(["benchmark", "machine", "--dev"])
.args(["--verify-duration", "0.1", "--disk-duration", "0.1"])
.args([
"--verify-duration",
"0.1",
"--disk-duration",
"0.1",
"--memory-duration",
"0.1",
"--hash-duration",
"0.1",
])
// Make it succeed.
.args(["--allow-fail"])
.status()
Expand All @@ -41,7 +50,18 @@ fn benchmark_machine_works() {
fn benchmark_machine_fails_with_slow_hardware() {
let output = Command::new(cargo_bin("substrate"))
.args(["benchmark", "machine", "--dev"])
.args(["--verify-duration", "0.1", "--disk-duration", "2", "--tolerance", "0"])
.args([
"--verify-duration",
"1.0",
"--disk-duration",
"2",
"--hash-duration",
"1.0",
"--memory-duration",
"1.0",
"--tolerance",
"0",
])
.output()
.unwrap();

Expand Down
51 changes: 27 additions & 24 deletions client/sysinfo/src/sysinfo.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ pub(crate) fn benchmark<E>(

let score = ((size * count) as f64 / elapsed.as_secs_f64()) / (1024.0 * 1024.0);
log::trace!(
"Calculated {} of {}MB/s in {} iterations in {}ms",
"Calculated {} of {:.2}MB/s in {} iterations in {}ms",
name,
score as u64,
score,
count,
elapsed.as_millis()
);
Expand Down Expand Up @@ -116,8 +116,12 @@ fn clobber_value<T>(input: &mut T) {
}
}

/// A default [`ExecutionLimit`] that can be used to call [`benchmark_cpu`].
pub const DEFAULT_CPU_EXECUTION_LIMIT: ExecutionLimit =
ExecutionLimit::Both { max_iterations: 4 * 1024, max_duration: Duration::from_millis(100) };

// This benchmarks the CPU speed as measured by calculating BLAKE2b-256 hashes, in MB/s.
pub fn benchmark_cpu() -> u64 {
pub fn benchmark_cpu(limit: ExecutionLimit) -> f64 {
// In general the results of this benchmark are somewhat sensitive to how much
// data we hash at the time. The smaller this is the *less* MB/s we can hash,
// the bigger this is the *more* MB/s we can hash, up until a certain point
Expand All @@ -131,8 +135,6 @@ pub fn benchmark_cpu() -> u64 {
// picked in such a way as to still measure how fast the hasher is at hashing,
// but without hitting its theoretical maximum speed.
const SIZE: usize = 32 * 1024;
const MAX_ITERATIONS: usize = 4 * 1024;
const MAX_DURATION: Duration = Duration::from_millis(100);

let mut buffer = Vec::new();
buffer.resize(SIZE, 0x66);
Expand All @@ -146,25 +148,27 @@ pub fn benchmark_cpu() -> u64 {
Ok(())
};

benchmark("CPU score", SIZE, MAX_ITERATIONS, MAX_DURATION, run)
.expect("benchmark cannot fail; qed") as u64
benchmark("CPU score", SIZE, limit.max_iterations(), limit.max_duration(), run)
.expect("benchmark cannot fail; qed")
}

/// A default [`ExecutionLimit`] that can be used to call [`benchmark_memory`].
pub const DEFAULT_MEMORY_EXECUTION_LIMIT: ExecutionLimit =
ExecutionLimit::Both { max_iterations: 32, max_duration: Duration::from_millis(100) };

// This benchmarks the effective `memcpy` memory bandwidth available in MB/s.
//
// It doesn't technically measure the absolute maximum memory bandwidth available,
// but that's fine, because real code most of the time isn't optimized to take
// advantage of the full memory bandwidth either.
pub fn benchmark_memory() -> u64 {
pub fn benchmark_memory(limit: ExecutionLimit) -> f64 {
// Ideally this should be at least as big as the CPU's L3 cache,
// and it should be big enough so that the `memcpy` takes enough
// time to be actually measurable.
//
// As long as it's big enough increasing it further won't change
// the benchmark's results.
const SIZE: usize = 64 * 1024 * 1024;
const MAX_ITERATIONS: usize = 32;
const MAX_DURATION: Duration = Duration::from_millis(100);

let mut src = Vec::new();
let mut dst = Vec::new();
Expand Down Expand Up @@ -192,8 +196,8 @@ pub fn benchmark_memory() -> u64 {
Ok(())
};

benchmark("memory score", SIZE, MAX_ITERATIONS, MAX_DURATION, run)
.expect("benchmark cannot fail; qed") as u64
benchmark("memory score", SIZE, limit.max_iterations(), limit.max_duration(), run)
.expect("benchmark cannot fail; qed")
}

struct TemporaryFile {
Expand Down Expand Up @@ -249,7 +253,7 @@ pub const DEFAULT_DISK_EXECUTION_LIMIT: ExecutionLimit =
pub fn benchmark_disk_sequential_writes(
limit: ExecutionLimit,
directory: &Path,
) -> Result<u64, String> {
) -> Result<f64, String> {
const SIZE: usize = 64 * 1024 * 1024;

let buffer = random_data(SIZE);
Expand Down Expand Up @@ -286,13 +290,12 @@ pub fn benchmark_disk_sequential_writes(
limit.max_duration(),
run,
)
.map(|s| s as u64)
}

pub fn benchmark_disk_random_writes(
limit: ExecutionLimit,
directory: &Path,
) -> Result<u64, String> {
) -> Result<f64, String> {
const SIZE: usize = 64 * 1024 * 1024;

let buffer = random_data(SIZE);
Expand Down Expand Up @@ -353,7 +356,6 @@ pub fn benchmark_disk_random_writes(
limit.max_duration(),
run,
)
.map(|s| s as u64)
}

/// Benchmarks the verification speed of sr25519 signatures.
Expand Down Expand Up @@ -400,8 +402,8 @@ pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> f64 {
pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench {
#[allow(unused_mut)]
let mut hwbench = HwBench {
cpu_hashrate_score: benchmark_cpu(),
memory_memcpy_score: benchmark_memory(),
cpu_hashrate_score: benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT) as u64,
memory_memcpy_score: benchmark_memory(DEFAULT_MEMORY_EXECUTION_LIMIT) as u64,
disk_sequential_write_score: None,
disk_random_write_score: None,
};
Expand All @@ -410,7 +412,7 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench {
hwbench.disk_sequential_write_score =
match benchmark_disk_sequential_writes(DEFAULT_DISK_EXECUTION_LIMIT, scratch_directory)
{
Ok(score) => Some(score),
Ok(score) => Some(score as u64),
Err(error) => {
log::warn!("Failed to run the sequential write disk benchmark: {}", error);
None
Expand All @@ -419,7 +421,7 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench {

hwbench.disk_random_write_score =
match benchmark_disk_random_writes(DEFAULT_DISK_EXECUTION_LIMIT, scratch_directory) {
Ok(score) => Some(score),
Ok(score) => Some(score as u64),
Err(error) => {
log::warn!("Failed to run the random write disk benchmark: {}", error);
None
Expand Down Expand Up @@ -448,26 +450,27 @@ mod tests {

#[test]
fn test_benchmark_cpu() {
assert_ne!(benchmark_cpu(), 0);
assert!(benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT) > 0.0);
}

#[test]
fn test_benchmark_memory() {
assert_ne!(benchmark_memory(), 0);
assert!(benchmark_memory(DEFAULT_MEMORY_EXECUTION_LIMIT) > 0.0);
}

#[test]
fn test_benchmark_disk_sequential_writes() {
assert!(
benchmark_disk_sequential_writes(DEFAULT_DISK_EXECUTION_LIMIT, "./".as_ref()).unwrap() >
0
0.0
);
}

#[test]
fn test_benchmark_disk_random_writes() {
assert!(
benchmark_disk_random_writes(DEFAULT_DISK_EXECUTION_LIMIT, "./".as_ref()).unwrap() > 0
benchmark_disk_random_writes(DEFAULT_DISK_EXECUTION_LIMIT, "./".as_ref()).unwrap() >
0.0
);
}

Expand Down
16 changes: 13 additions & 3 deletions utils/frame/benchmarking-cli/src/machine/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,17 @@ pub struct MachineCmd {
pub tolerance: f64,

/// Time limit for the verification benchmark.
#[clap(long, default_value = "2.0", value_name = "SECONDS")]
#[clap(long, default_value = "5.0", value_name = "SECONDS")]
pub verify_duration: f32,

/// Time limit for the hash function benchmark.
#[clap(long, default_value = "5.0", value_name = "SECONDS")]
pub hash_duration: f32,

/// Time limit for the memory benchmark.
#[clap(long, default_value = "5.0", value_name = "SECONDS")]
pub memory_duration: f32,

/// Time limit for each disk benchmark.
#[clap(long, default_value = "5.0", value_name = "SECONDS")]
pub disk_duration: f32,
Expand Down Expand Up @@ -134,11 +142,13 @@ impl MachineCmd {
fn measure(&self, metric: &Metric, dir: &Path) -> Result<Throughput> {
let verify_limit = ExecutionLimit::from_secs_f32(self.verify_duration);
let disk_limit = ExecutionLimit::from_secs_f32(self.disk_duration);
let hash_limit = ExecutionLimit::from_secs_f32(self.hash_duration);
let memory_limit = ExecutionLimit::from_secs_f32(self.memory_duration);

let score = match metric {
Metric::Blake2256 => Throughput::MiBs(benchmark_cpu() as f64),
Metric::Blake2256 => Throughput::MiBs(benchmark_cpu(hash_limit) as f64),
Metric::Sr25519Verify => Throughput::MiBs(benchmark_sr25519_verify(verify_limit)),
Metric::MemCopy => Throughput::MiBs(benchmark_memory() as f64),
Metric::MemCopy => Throughput::MiBs(benchmark_memory(memory_limit) as f64),
Metric::DiskSeqWrite =>
Throughput::MiBs(benchmark_disk_sequential_writes(disk_limit, dir)? as f64),
Metric::DiskRndWrite =>
Expand Down

0 comments on commit c44cd63

Please sign in to comment.