diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 4da32f5cb..37084ffc1 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -18,6 +18,12 @@ pub struct BenchmarkRun { /// Time spent in kernel mode pub system_time: Second, + + /// Maximum memory usage of the process, in bytes + pub memory_usage_byte: u64, + + /// Exit codes of the process + pub exit_code: Option, } /// Set of values that will be exported. @@ -36,13 +42,6 @@ pub struct BenchmarkResult { /// All run time measurements pub runs: Vec, - /// Maximum memory usage of the process, in bytes - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_usage_byte: Option>, - - /// Exit codes of all command invocations - pub exit_codes: Vec>, - /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 2beb870a0..208f6ad90 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -448,15 +448,25 @@ impl<'a> Benchmark<'a> { command_with_unused_parameters: self.command.get_name_with_unused_parameters(), runs: times_real .iter() - .zip(times_user.iter().zip(times_system.iter())) - .map(|(wall_clock_time, (user_time, system_time))| BenchmarkRun { - wall_clock_time: *wall_clock_time, - user_time: *user_time, - system_time: *system_time, - }) + .zip(times_user.iter()) + .zip(times_system.iter()) + .zip(memory_usage_byte.iter()) + .zip(exit_codes.iter()) + .map( + |( + (((wall_clock_time, user_time), system_time), memory_usage_byte), + exit_code, + )| { + BenchmarkRun { + wall_clock_time: *wall_clock_time, + user_time: *user_time, + system_time: *system_time, + memory_usage_byte: *memory_usage_byte, + exit_code: *exit_code, + } + }, + ) .collect(), - memory_usage_byte: Some(memory_usage_byte), - exit_codes, parameters: self .command .get_parameters() diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 7b03df757..6dfb67674 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -131,9 +131,9 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { wall_clock_time: mean, user_time: mean, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }], - memory_usage_byte: None, - exit_codes: Vec::new(), parameters: BTreeMap::new(), } } diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 152e00c5b..473650aee 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -193,29 +193,25 @@ fn scheduler_basic() -> Result<()> { - wall_clock_time: 0.123 user_time: 0 system_time: 0 + memory_usage_byte: 0 + exit_code: 0 - wall_clock_time: 0.123 user_time: 0 system_time: 0 - memory_usage_byte: - - 0 - - 0 - exit_codes: - - 0 - - 0 + memory_usage_byte: 0 + exit_code: 0 - command: sleep 0.456 runs: - wall_clock_time: 0.456 user_time: 0 system_time: 0 + memory_usage_byte: 0 + exit_code: 0 - wall_clock_time: 0.456 user_time: 0 system_time: 0 - memory_usage_byte: - - 0 - - 0 - exit_codes: - - 0 - - 0 + memory_usage_byte: 0 + exit_code: 0 "#); Ok(()) diff --git a/src/export/csv.rs b/src/export/csv.rs index 52bf17b60..5054cf0c3 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -77,20 +77,24 @@ fn test_csv() { wall_clock_time: 7.0, user_time: 7.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 8.0, user_time: 8.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 12.0, user_time: 12.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); @@ -106,20 +110,24 @@ fn test_csv() { wall_clock_time: 17.0, user_time: 17.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 18.0, user_time: 18.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 19.0, user_time: 19.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); diff --git a/src/export/tests.rs b/src/export/tests.rs index 76026958e..8fdaf5612 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -32,20 +32,24 @@ fn test_markup_export_auto_ms() { wall_clock_time: 0.09, user_time: 0.09, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.10, user_time: 0.10, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.14, user_time: 0.14, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -56,20 +60,24 @@ fn test_markup_export_auto_ms() { wall_clock_time: 2.0, user_time: 2.0, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 3.0, user_time: 3.0, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 4.0, user_time: 4.0, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -125,20 +133,24 @@ fn test_markup_export_auto_s() { wall_clock_time: 2.1, user_time: 2.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.2, user_time: 2.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.3, user_time: 2.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -149,20 +161,24 @@ fn test_markup_export_auto_s() { wall_clock_time: 0.1, user_time: 0.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.2, user_time: 0.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.3, user_time: 0.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -218,20 +234,24 @@ fn test_markup_export_manual_ms() { wall_clock_time: 2.1, user_time: 2.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.2, user_time: 2.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.3, user_time: 2.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -242,20 +262,24 @@ fn test_markup_export_manual_ms() { wall_clock_time: 0.1, user_time: 0.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.2, user_time: 0.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.3, user_time: 0.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -310,20 +334,24 @@ fn test_markup_export_manual_s() { wall_clock_time: 2.01, user_time: 2.01, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.02, user_time: 2.02, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.03, user_time: 2.03, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -334,20 +362,24 @@ fn test_markup_export_manual_s() { wall_clock_time: 0.11, user_time: 0.11, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.12, user_time: 0.12, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.13, user_time: 0.13, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; diff --git a/src/timer/mod.rs b/src/timer/mod.rs index b2cd2654c..8251fbba8 100644 --- a/src/timer/mod.rs +++ b/src/timer/mod.rs @@ -24,19 +24,6 @@ use std::process::{ChildStdout, Command, ExitStatus}; use anyhow::Result; -#[cfg(not(windows))] -#[derive(Debug, Copy, Clone)] -struct CPUTimes { - /// Total amount of time spent executing in user mode - pub user_usec: i64, - - /// Total amount of time spent executing in kernel mode - pub system_usec: i64, - - /// Maximum amount of memory used by the process, in bytes - pub memory_usage_byte: u64, -} - /// Used to indicate the result of running a command #[derive(Debug, Copy, Clone)] pub struct TimerResult { @@ -106,10 +93,8 @@ pub fn execute_and_measure(mut command: Command) -> Result { discard(output); } - let status = child.wait()?; - + let (status, time_user, time_system, memory_usage_byte) = cpu_timer.stop(child)?; let time_real = wallclock_timer.stop(); - let (time_user, time_system, memory_usage_byte) = cpu_timer.stop(); Ok(TimerResult { time_real, diff --git a/src/timer/unix_timer.rs b/src/timer/unix_timer.rs index 34e5325cb..b76438c3d 100644 --- a/src/timer/unix_timer.rs +++ b/src/timer/unix_timer.rs @@ -1,106 +1,80 @@ #![cfg(not(windows))] use std::convert::TryFrom; -use std::mem; +use std::io; +use std::mem::MaybeUninit; +use std::os::unix::process::ExitStatusExt; +use std::process::{Child, ExitStatus}; + +use anyhow::Result; -use crate::timer::CPUTimes; use crate::util::units::Second; #[derive(Debug, Copy, Clone)] -pub struct CPUInterval { +struct ResourceUsage { /// Total amount of time spent executing in user mode - pub user: Second, + pub user_usec: Second, /// Total amount of time spent executing in kernel mode - pub system: Second, -} + pub system_usec: Second, -pub struct CPUTimer { - start_cpu: CPUTimes, + /// Maximum amount of memory used by the process, in bytes + pub memory_usage_byte: i64, } -impl CPUTimer { - pub fn start() -> Self { - CPUTimer { - start_cpu: get_cpu_times(), - } - } - - pub fn stop(&self) -> (Second, Second, u64) { - let end_cpu = get_cpu_times(); - let cpu_interval = cpu_time_interval(&self.start_cpu, &end_cpu); - ( - cpu_interval.user, - cpu_interval.system, - end_cpu.memory_usage_byte, - ) - } +#[allow(clippy::useless_conversion)] +fn timeval_to_seconds(tv: libc::timeval) -> Second { + const MICROSEC_PER_SEC: i64 = 1000 * 1000; + (i64::from(tv.tv_sec) * MICROSEC_PER_SEC + i64::from(tv.tv_usec)) as f64 * 1e-6 } -/// Read CPU execution times ('user' and 'system') -fn get_cpu_times() -> CPUTimes { - use libc::{getrusage, rusage, RUSAGE_CHILDREN}; +#[allow(clippy::useless_conversion)] +fn wait4(mut child: Child) -> io::Result<(ExitStatus, ResourceUsage)> { + drop(child.stdin.take()); - let result: rusage = unsafe { - let mut buf = mem::zeroed(); - let success = getrusage(RUSAGE_CHILDREN, &mut buf); - assert_eq!(0, success); - buf - }; + let pid = child.id() as i32; + let mut status = 0; + let mut rusage = MaybeUninit::zeroed(); - const MICROSEC_PER_SEC: i64 = 1000 * 1000; + let result = unsafe { libc::wait4(pid, &mut status, 0, rusage.as_mut_ptr()) }; - // Linux and *BSD return the value in KibiBytes, Darwin flavors in bytes - let max_rss_byte = if cfg!(target_os = "macos") || cfg!(target_os = "ios") { - result.ru_maxrss + if result < 0 { + Err(io::Error::last_os_error()) } else { - result.ru_maxrss * 1024 - }; - - #[allow(clippy::useless_conversion)] - CPUTimes { - user_usec: i64::from(result.ru_utime.tv_sec) * MICROSEC_PER_SEC - + i64::from(result.ru_utime.tv_usec), - system_usec: i64::from(result.ru_stime.tv_sec) * MICROSEC_PER_SEC - + i64::from(result.ru_stime.tv_usec), - memory_usage_byte: u64::try_from(max_rss_byte).unwrap_or(0), + let rusage = unsafe { rusage.assume_init() }; + + let memory_usage_byte = if cfg!(target_os = "macos") || cfg!(target_os = "ios") { + // Linux and *BSD return the value in KibiBytes, Darwin flavors in bytes + rusage.ru_maxrss + } else { + rusage.ru_maxrss * 1024 + }; + + Ok(( + ExitStatus::from_raw(status), + ResourceUsage { + user_usec: timeval_to_seconds(rusage.ru_utime), + system_usec: timeval_to_seconds(rusage.ru_stime), + memory_usage_byte: memory_usage_byte.into(), + }, + )) } } -/// Compute the time intervals in between two `CPUTimes` snapshots -fn cpu_time_interval(start: &CPUTimes, end: &CPUTimes) -> CPUInterval { - CPUInterval { - user: ((end.user_usec - start.user_usec) as f64) * 1e-6, - system: ((end.system_usec - start.system_usec) as f64) * 1e-6, +pub struct CPUTimer {} + +impl CPUTimer { + pub fn start() -> Self { + Self {} } -} -#[cfg(test)] -use approx::assert_relative_eq; - -#[test] -fn test_cpu_time_interval() { - let t_a = CPUTimes { - user_usec: 12345, - system_usec: 54321, - memory_usage_byte: 0, - }; - - let t_b = CPUTimes { - user_usec: 20000, - system_usec: 70000, - memory_usage_byte: 0, - }; - - let t_zero = cpu_time_interval(&t_a, &t_a); - assert!(t_zero.user.abs() < f64::EPSILON); - assert!(t_zero.system.abs() < f64::EPSILON); - - let t_ab = cpu_time_interval(&t_a, &t_b); - assert_relative_eq!(0.007655, t_ab.user); - assert_relative_eq!(0.015679, t_ab.system); - - let t_ba = cpu_time_interval(&t_b, &t_a); - assert_relative_eq!(-0.007655, t_ba.user); - assert_relative_eq!(-0.015679, t_ba.system); + pub fn stop(&self, child: Child) -> Result<(ExitStatus, Second, Second, u64)> { + let (status, usage) = wait4(child)?; + Ok(( + status, + usage.user_usec, + usage.system_usec, + u64::try_from(usage.memory_usage_byte).unwrap_or(0), + )) + } } diff --git a/src/timer/windows_timer.rs b/src/timer/windows_timer.rs index 04dc9fcf1..8c63acb2f 100644 --- a/src/timer/windows_timer.rs +++ b/src/timer/windows_timer.rs @@ -1,7 +1,10 @@ #![cfg(windows)] #![warn(unsafe_op_in_unsafe_fn)] -use std::{mem, os::windows::io::AsRawHandle, process, ptr}; +use std::process::{self, Child, ExitStatus}; +use std::{mem, os::windows::io::AsRawHandle, ptr}; + +use anyhow::Result; use windows_sys::Win32::{ Foundation::{CloseHandle, HANDLE}, @@ -86,7 +89,9 @@ impl CPUTimer { Self { job_object } } - pub fn stop(&self) -> (Second, Second, u64) { + pub fn stop(&self, mut child: Child) -> Result<(ExitStatus, Second, Second, u64)> { + let status = child.wait()?; + let mut job_object_info = mem::MaybeUninit::::uninit(); @@ -114,9 +119,9 @@ impl CPUTimer { // for all active processes associated with the job, as well as all terminated // processes no longer associated with the job, in 100-nanosecond ticks." let kernel: i64 = job_object_info.TotalKernelTime / HUNDRED_NS_PER_MS; - (user as f64 * 1e-6, kernel as f64 * 1e-6, 0) + Ok((status, user as f64 * 1e-6, kernel as f64 * 1e-6, 0)) } else { - (0.0, 0.0, 0) + Ok((status, 0.0, 0.0, 0)) } } }