diff options
author | Jakub Kotur <qtr@google.com> | 2020-12-21 17:28:14 +0100 |
---|---|---|
committer | Jakub Kotur <qtr@google.com> | 2021-03-05 15:05:06 +0100 |
commit | 704f579139cd14c990899a887026adce4b6fb0ac (patch) | |
tree | 8ab32923fe4a5e70e694e6147ea7f6783bc1b04b /src | |
parent | 71f53f93f889decb4b2faafb4eaea7b0e9c70722 (diff) | |
download | criterion-704f579139cd14c990899a887026adce4b6fb0ac.tar.gz |
Initial import of criterion-0.3.3.
Bug: 155309706
Change-Id: I8d6ee6f1995361f33b3b63527d1236f13de1ab0c
Diffstat (limited to 'src')
57 files changed, 12727 insertions, 0 deletions
diff --git a/src/analysis/compare.rs b/src/analysis/compare.rs new file mode 100755 index 0000000..a49407d --- /dev/null +++ b/src/analysis/compare.rs @@ -0,0 +1,143 @@ +use crate::stats::univariate::Sample; +use crate::stats::univariate::{self, mixed}; +use crate::stats::Distribution; + +use crate::benchmark::BenchmarkConfig; +use crate::error::Result; +use crate::estimate::{ + build_change_estimates, ChangeDistributions, ChangeEstimates, ChangePointEstimates, Estimates, +}; +use crate::measurement::Measurement; +use crate::report::BenchmarkId; +use crate::{fs, Criterion, SavedSample}; + +// Common comparison procedure +#[cfg_attr(feature = "cargo-clippy", allow(clippy::type_complexity))] +pub(crate) fn common<M: Measurement>( + id: &BenchmarkId, + avg_times: &Sample<f64>, + config: &BenchmarkConfig, + criterion: &Criterion<M>, +) -> Result<( + f64, + Distribution<f64>, + ChangeEstimates, + ChangeDistributions, + Vec<f64>, + Vec<f64>, + Vec<f64>, + Estimates, +)> { + let mut sample_file = criterion.output_directory.clone(); + sample_file.push(id.as_directory_name()); + sample_file.push(&criterion.baseline_directory); + sample_file.push("sample.json"); + let sample: SavedSample = fs::load(&sample_file)?; + let SavedSample { iters, times, .. } = sample; + + let mut estimates_file = criterion.output_directory.clone(); + estimates_file.push(id.as_directory_name()); + estimates_file.push(&criterion.baseline_directory); + estimates_file.push("estimates.json"); + let base_estimates: Estimates = fs::load(&estimates_file)?; + + let base_avg_times: Vec<f64> = iters + .iter() + .zip(times.iter()) + .map(|(iters, elapsed)| elapsed / iters) + .collect(); + let base_avg_time_sample = Sample::new(&base_avg_times); + + let mut change_dir = criterion.output_directory.clone(); + change_dir.push(id.as_directory_name()); + change_dir.push("change"); + fs::mkdirp(&change_dir)?; + let (t_statistic, t_distribution) = t_test(avg_times, base_avg_time_sample, config); + + let (estimates, relative_distributions) = + estimates(id, avg_times, base_avg_time_sample, config, criterion); + Ok(( + t_statistic, + t_distribution, + estimates, + relative_distributions, + iters, + times, + base_avg_times.clone(), + base_estimates, + )) +} + +// Performs a two sample t-test +fn t_test( + avg_times: &Sample<f64>, + base_avg_times: &Sample<f64>, + config: &BenchmarkConfig, +) -> (f64, Distribution<f64>) { + let nresamples = config.nresamples; + + let t_statistic = avg_times.t(base_avg_times); + let t_distribution = elapsed!( + "Bootstrapping the T distribution", + mixed::bootstrap(avg_times, base_avg_times, nresamples, |a, b| (a.t(b),)) + ) + .0; + + // HACK: Filter out non-finite numbers, which can happen sometimes when sample size is very small. + // Downstream code doesn't like non-finite values here. + let t_distribution = Distribution::from( + t_distribution + .iter() + .filter(|a| a.is_finite()) + .cloned() + .collect::<Vec<_>>() + .into_boxed_slice(), + ); + + (t_statistic, t_distribution) +} + +// Estimates the relative change in the statistics of the population +fn estimates<M: Measurement>( + id: &BenchmarkId, + avg_times: &Sample<f64>, + base_avg_times: &Sample<f64>, + config: &BenchmarkConfig, + criterion: &Criterion<M>, +) -> (ChangeEstimates, ChangeDistributions) { + fn stats(a: &Sample<f64>, b: &Sample<f64>) -> (f64, f64) { + ( + a.mean() / b.mean() - 1., + a.percentiles().median() / b.percentiles().median() - 1., + ) + } + + let cl = config.confidence_level; + let nresamples = config.nresamples; + + let (dist_mean, dist_median) = elapsed!( + "Bootstrapping the relative statistics", + univariate::bootstrap(avg_times, base_avg_times, nresamples, stats) + ); + + let distributions = ChangeDistributions { + mean: dist_mean, + median: dist_median, + }; + + let (mean, median) = stats(avg_times, base_avg_times); + let points = ChangePointEstimates { mean, median }; + + let estimates = build_change_estimates(&distributions, &points, cl); + + { + log_if_err!({ + let mut estimates_path = criterion.output_directory.clone(); + estimates_path.push(id.as_directory_name()); + estimates_path.push("change"); + estimates_path.push("estimates.json"); + fs::save(&estimates, &estimates_path) + }); + } + (estimates, distributions) +} diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs new file mode 100755 index 0000000..caa948d --- /dev/null +++ b/src/analysis/mod.rs @@ -0,0 +1,358 @@ +use std::path::Path; + +use crate::stats::bivariate::regression::Slope; +use crate::stats::bivariate::Data; +use crate::stats::univariate::outliers::tukey; +use crate::stats::univariate::Sample; +use crate::stats::{Distribution, Tails}; + +use crate::benchmark::BenchmarkConfig; +use crate::connection::OutgoingMessage; +use crate::estimate::{ + build_estimates, ConfidenceInterval, Distributions, Estimate, Estimates, PointEstimates, +}; +use crate::fs; +use crate::measurement::Measurement; +use crate::report::{BenchmarkId, ReportContext}; +use crate::routine::Routine; +use crate::{Baseline, Criterion, SavedSample, Throughput}; + +macro_rules! elapsed { + ($msg:expr, $block:expr) => {{ + let start = ::std::time::Instant::now(); + let out = $block; + let elapsed = &start.elapsed(); + + info!( + "{} took {}", + $msg, + crate::format::time(crate::DurationExt::to_nanos(elapsed) as f64) + ); + + out + }}; +} + +mod compare; + +// Common analysis procedure +pub(crate) fn common<M: Measurement, T: ?Sized>( + id: &BenchmarkId, + routine: &mut dyn Routine<M, T>, + config: &BenchmarkConfig, + criterion: &Criterion<M>, + report_context: &ReportContext, + parameter: &T, + throughput: Option<Throughput>, +) { + criterion.report.benchmark_start(id, report_context); + + if let Baseline::Compare = criterion.baseline { + if !base_dir_exists( + id, + &criterion.baseline_directory, + &criterion.output_directory, + ) { + panic!(format!( + "Baseline '{base}' must exist before comparison is allowed; try --save-baseline {base}", + base=criterion.baseline_directory, + )); + } + } + + let (sampling_mode, iters, times); + if let Some(baseline) = &criterion.load_baseline { + let mut sample_path = criterion.output_directory.clone(); + sample_path.push(id.as_directory_name()); + sample_path.push(baseline); + sample_path.push("sample.json"); + let loaded = fs::load::<SavedSample, _>(&sample_path); + + match loaded { + Err(err) => panic!( + "Baseline '{base}' must exist before it can be loaded; try --save-baseline {base}. Error: {err}", + base = baseline, err = err + ), + Ok(samples) => { + sampling_mode = samples.sampling_mode; + iters = samples.iters.into_boxed_slice(); + times = samples.times.into_boxed_slice(); + } + } + } else { + let sample = routine.sample( + &criterion.measurement, + id, + config, + criterion, + report_context, + parameter, + ); + sampling_mode = sample.0; + iters = sample.1; + times = sample.2; + + if let Some(conn) = &criterion.connection { + conn.send(&OutgoingMessage::MeasurementComplete { + id: id.into(), + iters: &iters, + times: ×, + plot_config: (&report_context.plot_config).into(), + sampling_method: sampling_mode.into(), + benchmark_config: config.into(), + }) + .unwrap(); + + conn.serve_value_formatter(criterion.measurement.formatter()) + .unwrap(); + } + } + + criterion.report.analysis(id, report_context); + + let avg_times = iters + .iter() + .zip(times.iter()) + .map(|(&iters, &elapsed)| elapsed / iters) + .collect::<Vec<f64>>(); + let avg_times = Sample::new(&avg_times); + + if criterion.connection.is_none() && criterion.load_baseline.is_none() { + log_if_err!({ + let mut new_dir = criterion.output_directory.clone(); + new_dir.push(id.as_directory_name()); + new_dir.push("new"); + fs::mkdirp(&new_dir) + }); + } + + let data = Data::new(&iters, ×); + let labeled_sample = tukey::classify(avg_times); + if criterion.connection.is_none() { + log_if_err!({ + let mut tukey_file = criterion.output_directory.to_owned(); + tukey_file.push(id.as_directory_name()); + tukey_file.push("new"); + tukey_file.push("tukey.json"); + fs::save(&labeled_sample.fences(), &tukey_file) + }); + } + let (mut distributions, mut estimates) = estimates(avg_times, config); + if sampling_mode.is_linear() { + let (distribution, slope) = regression(&data, config); + + estimates.slope = Some(slope); + distributions.slope = Some(distribution); + } + + if criterion.connection.is_none() && criterion.load_baseline.is_none() { + log_if_err!({ + let mut sample_file = criterion.output_directory.clone(); + sample_file.push(id.as_directory_name()); + sample_file.push("new"); + sample_file.push("sample.json"); + fs::save( + &SavedSample { + sampling_mode, + iters: data.x().as_ref().to_vec(), + times: data.y().as_ref().to_vec(), + }, + &sample_file, + ) + }); + log_if_err!({ + let mut estimates_file = criterion.output_directory.clone(); + estimates_file.push(id.as_directory_name()); + estimates_file.push("new"); + estimates_file.push("estimates.json"); + fs::save(&estimates, &estimates_file) + }); + } + + let compare_data = if base_dir_exists( + id, + &criterion.baseline_directory, + &criterion.output_directory, + ) { + let result = compare::common(id, avg_times, config, criterion); + match result { + Ok(( + t_value, + t_distribution, + relative_estimates, + relative_distributions, + base_iter_counts, + base_sample_times, + base_avg_times, + base_estimates, + )) => { + let p_value = t_distribution.p_value(t_value, &Tails::Two); + Some(crate::report::ComparisonData { + p_value, + t_distribution, + t_value, + relative_estimates, + relative_distributions, + significance_threshold: config.significance_level, + noise_threshold: config.noise_threshold, + base_iter_counts, + base_sample_times, + base_avg_times, + base_estimates, + }) + } + Err(e) => { + crate::error::log_error(&e); + None + } + } + } else { + None + }; + + let measurement_data = crate::report::MeasurementData { + data: Data::new(&*iters, &*times), + avg_times: labeled_sample, + absolute_estimates: estimates, + distributions, + comparison: compare_data, + throughput, + }; + + criterion.report.measurement_complete( + id, + report_context, + &measurement_data, + criterion.measurement.formatter(), + ); + + if criterion.connection.is_none() && criterion.load_baseline.is_none() { + log_if_err!({ + let mut benchmark_file = criterion.output_directory.clone(); + benchmark_file.push(id.as_directory_name()); + benchmark_file.push("new"); + benchmark_file.push("benchmark.json"); + fs::save(&id, &benchmark_file) + }); + } + + if criterion.connection.is_none() { + if let Baseline::Save = criterion.baseline { + copy_new_dir_to_base( + id.as_directory_name(), + &criterion.baseline_directory, + &criterion.output_directory, + ); + } + } +} + +fn base_dir_exists(id: &BenchmarkId, baseline: &str, output_directory: &Path) -> bool { + let mut base_dir = output_directory.to_owned(); + base_dir.push(id.as_directory_name()); + base_dir.push(baseline); + base_dir.exists() +} + +// Performs a simple linear regression on the sample +fn regression( + data: &Data<'_, f64, f64>, + config: &BenchmarkConfig, +) -> (Distribution<f64>, Estimate) { + let cl = config.confidence_level; + + let distribution = elapsed!( + "Bootstrapped linear regression", + data.bootstrap(config.nresamples, |d| (Slope::fit(&d).0,)) + ) + .0; + + let point = Slope::fit(&data); + let (lb, ub) = distribution.confidence_interval(config.confidence_level); + let se = distribution.std_dev(None); + + ( + distribution, + Estimate { + confidence_interval: ConfidenceInterval { + confidence_level: cl, + lower_bound: lb, + upper_bound: ub, + }, + point_estimate: point.0, + standard_error: se, + }, + ) +} + +// Estimates the statistics of the population from the sample +fn estimates(avg_times: &Sample<f64>, config: &BenchmarkConfig) -> (Distributions, Estimates) { + fn stats(sample: &Sample<f64>) -> (f64, f64, f64, f64) { + let mean = sample.mean(); + let std_dev = sample.std_dev(Some(mean)); + let median = sample.percentiles().median(); + let mad = sample.median_abs_dev(Some(median)); + + (mean, std_dev, median, mad) + } + + let cl = config.confidence_level; + let nresamples = config.nresamples; + + let (mean, std_dev, median, mad) = stats(avg_times); + let points = PointEstimates { + mean, + median, + std_dev, + median_abs_dev: mad, + }; + + let (dist_mean, dist_stddev, dist_median, dist_mad) = elapsed!( + "Bootstrapping the absolute statistics.", + avg_times.bootstrap(nresamples, stats) + ); + + let distributions = Distributions { + mean: dist_mean, + slope: None, + median: dist_median, + median_abs_dev: dist_mad, + std_dev: dist_stddev, + }; + + let estimates = build_estimates(&distributions, &points, cl); + + (distributions, estimates) +} + +fn copy_new_dir_to_base(id: &str, baseline: &str, output_directory: &Path) { + let root_dir = Path::new(output_directory).join(id); + let base_dir = root_dir.join(baseline); + let new_dir = root_dir.join("new"); + + if !new_dir.exists() { + return; + }; + if !base_dir.exists() { + try_else_return!(fs::mkdirp(&base_dir)); + } + + // TODO: consider using walkdir or similar to generically copy. + try_else_return!(fs::cp( + &new_dir.join("estimates.json"), + &base_dir.join("estimates.json") + )); + try_else_return!(fs::cp( + &new_dir.join("sample.json"), + &base_dir.join("sample.json") + )); + try_else_return!(fs::cp( + &new_dir.join("tukey.json"), + &base_dir.join("tukey.json") + )); + try_else_return!(fs::cp( + &new_dir.join("benchmark.json"), + &base_dir.join("benchmark.json") + )); + try_else_return!(fs::cp(&new_dir.join("raw.csv"), &base_dir.join("raw.csv"))); +} diff --git a/src/benchmark.rs b/src/benchmark.rs new file mode 100755 index 0000000..a99750d --- /dev/null +++ b/src/benchmark.rs @@ -0,0 +1,611 @@ +use crate::analysis; +use crate::connection::OutgoingMessage; +use crate::measurement::{Measurement, WallTime}; +use crate::report::{BenchmarkId, ReportContext}; +use crate::routine::{Function, Routine}; +use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput}; +use std::cell::RefCell; +use std::fmt::Debug; +use std::marker::Sized; +use std::time::Duration; + +// TODO: Move the benchmark config stuff to a separate module for easier use. + +/// Struct containing all of the configuration options for a benchmark. +pub struct BenchmarkConfig { + pub confidence_level: f64, + pub measurement_time: Duration, + pub noise_threshold: f64, + pub nresamples: usize, + pub sample_size: usize, + pub significance_level: f64, + pub warm_up_time: Duration, + pub sampling_mode: SamplingMode, +} + +/// Struct representing a partially-complete per-benchmark configuration. +#[derive(Clone)] +pub(crate) struct PartialBenchmarkConfig { + pub(crate) confidence_level: Option<f64>, + pub(crate) measurement_time: Option<Duration>, + pub(crate) noise_threshold: Option<f64>, + pub(crate) nresamples: Option<usize>, + pub(crate) sample_size: Option<usize>, + pub(crate) significance_level: Option<f64>, + pub(crate) warm_up_time: Option<Duration>, + pub(crate) sampling_mode: Option<SamplingMode>, + pub(crate) plot_config: PlotConfiguration, +} + +impl Default for PartialBenchmarkConfig { + fn default() -> Self { + PartialBenchmarkConfig { + confidence_level: None, + measurement_time: None, + noise_threshold: None, + nresamples: None, + sample_size: None, + significance_level: None, + warm_up_time: None, + plot_config: PlotConfiguration::default(), + sampling_mode: None, + } + } +} + +impl PartialBenchmarkConfig { + pub(crate) fn to_complete(&self, defaults: &BenchmarkConfig) -> BenchmarkConfig { + BenchmarkConfig { + confidence_level: self.confidence_level.unwrap_or(defaults.confidence_level), + measurement_time: self.measurement_time.unwrap_or(defaults.measurement_time), + noise_threshold: self.noise_threshold.unwrap_or(defaults.noise_threshold), + nresamples: self.nresamples.unwrap_or(defaults.nresamples), + sample_size: self.sample_size.unwrap_or(defaults.sample_size), + significance_level: self + .significance_level + .unwrap_or(defaults.significance_level), + warm_up_time: self.warm_up_time.unwrap_or(defaults.warm_up_time), + sampling_mode: self.sampling_mode.unwrap_or(defaults.sampling_mode), + } + } +} + +pub(crate) struct NamedRoutine<T, M: Measurement = WallTime> { + pub id: String, + pub(crate) f: Box<RefCell<dyn Routine<M, T>>>, +} + +/// Structure representing a benchmark (or group of benchmarks) +/// which take one parameter. +#[doc(hidden)] +pub struct ParameterizedBenchmark<T: Debug, M: Measurement = WallTime> { + config: PartialBenchmarkConfig, + values: Vec<T>, + routines: Vec<NamedRoutine<T, M>>, + throughput: Option<Box<dyn Fn(&T) -> Throughput>>, +} + +/// Structure representing a benchmark (or group of benchmarks) +/// which takes no parameters. +#[doc(hidden)] +pub struct Benchmark<M: Measurement = WallTime> { + config: PartialBenchmarkConfig, + routines: Vec<NamedRoutine<(), M>>, + throughput: Option<Throughput>, +} + +/// Common trait for `Benchmark` and `ParameterizedBenchmark`. Not intended to be +/// used outside of Criterion.rs. +#[doc(hidden)] +pub trait BenchmarkDefinition<M: Measurement = WallTime>: Sized { + #[doc(hidden)] + fn run(self, group_id: &str, c: &mut Criterion<M>); +} + +macro_rules! benchmark_config { + ($type:tt) => { + /// Changes the size of the sample for this benchmark + /// + /// A bigger sample should yield more accurate results if paired with a sufficiently large + /// measurement time. + /// + /// Sample size must be at least 10. + /// + /// # Panics + /// + /// Panics if n < 10. + pub fn sample_size(mut self, n: usize) -> Self { + assert!(n >= 10); + + self.config.sample_size = Some(n); + self + } + + /// Changes the warm up time for this benchmark + /// + /// # Panics + /// + /// Panics if the input duration is zero + pub fn warm_up_time(mut self, dur: Duration) -> Self { + assert!(dur.to_nanos() > 0); + + self.config.warm_up_time = Some(dur); + self + } + + /// Changes the target measurement time for this benchmark. Criterion will attempt + /// to spent approximately this amount of time measuring the benchmark. + /// With a longer time, the measurement will become more resilient to transitory peak loads + /// caused by external programs. + /// + /// # Panics + /// + /// Panics if the input duration in zero + pub fn measurement_time(mut self, dur: Duration) -> Self { + assert!(dur.to_nanos() > 0); + + self.config.measurement_time = Some(dur); + self + } + + /// Changes the number of resamples for this benchmark + /// + /// Number of resamples to use for the + /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling) + /// + /// A larger number of resamples reduces the random sampling errors, which are inherent to the + /// bootstrap method, but also increases the analysis time. + /// + /// # Panics + /// + /// Panics if the number of resamples is set to zero + pub fn nresamples(mut self, n: usize) -> Self { + assert!(n > 0); + if n <= 1000 { + println!("\nWarning: It is not recommended to reduce nresamples below 1000."); + } + + self.config.nresamples = Some(n); + self + } + + /// Changes the default noise threshold for this benchmark. The noise threshold + /// is used to filter out small changes in performance, even if they are statistically + /// significant. Sometimes benchmarking the same code twice will result in small but + /// statistically significant differences solely because of noise. This provides a way to filter + /// out some of these false positives at the cost of making it harder to detect small changes + /// to the true performance of the benchmark. + /// + /// The default is 0.01, meaning that changes smaller than 1% will be ignored. + /// + /// # Panics + /// + /// Panics if the threshold is set to a negative value + pub fn noise_threshold(mut self, threshold: f64) -> Self { + assert!(threshold >= 0.0); + + self.config.noise_threshold = Some(threshold); + self + } + + /// Changes the default confidence level for this benchmark. The confidence + /// level is the desired probability that the true runtime lies within the estimated + /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is + /// 0.95, meaning that the confidence interval should capture the true value 95% of the time. + /// + /// # Panics + /// + /// Panics if the confidence level is set to a value outside the `(0, 1)` range + pub fn confidence_level(mut self, cl: f64) -> Self { + assert!(cl > 0.0 && cl < 1.0); + if cl < 0.5 { + println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); + } + + self.config.confidence_level = Some(cl); + self + } + + /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance) + /// for this benchmark. This is used to perform a + /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if + /// the measurements from this run are different from the measured performance of the last run. + /// The significance level is the desired probability that two measurements of identical code + /// will be considered 'different' due to noise in the measurements. The default value is 0.05, + /// meaning that approximately 5% of identical benchmarks will register as different due to + /// noise. + /// + /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase + /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to + /// detect small but real changes in the performance. By setting the significance level + /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also + /// report more spurious differences. + /// + /// See also the noise threshold setting. + /// + /// # Panics + /// + /// Panics if the significance level is set to a value outside the `(0, 1)` range + pub fn significance_level(mut self, sl: f64) -> Self { + assert!(sl > 0.0 && sl < 1.0); + + self.config.significance_level = Some(sl); + self + } + + /// Changes the plot configuration for this benchmark. + pub fn plot_config(mut self, new_config: PlotConfiguration) -> Self { + self.config.plot_config = new_config; + self + } + + /// Changes the sampling mode for this benchmark. + pub fn sampling_mode(mut self, new_mode: SamplingMode) -> Self { + self.config.sampling_mode = Some(new_mode); + self + } + }; +} + +impl<M> Benchmark<M> +where + M: Measurement + 'static, +{ + benchmark_config!(Benchmark); + + /// Create a new benchmark group and adds the given function to it. + /// + /// # Example + /// + /// ```rust + /// # #[macro_use] extern crate criterion; + /// # use criterion::*; + /// + /// fn bench(c: &mut Criterion) { + /// // One-time setup goes here + /// c.bench( + /// "my_group", + /// Benchmark::new("my_function", |b| b.iter(|| { + /// // Code to benchmark goes here + /// })), + /// ); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + pub fn new<S, F>(id: S, f: F) -> Benchmark<M> + where + S: Into<String>, + F: FnMut(&mut Bencher<'_, M>) + 'static, + { + Benchmark { + config: PartialBenchmarkConfig::default(), + routines: vec![], + throughput: None, + } + .with_function(id, f) + } + + /// Add a function to the benchmark group. + pub fn with_function<S, F>(mut self, id: S, mut f: F) -> Benchmark<M> + where + S: Into<String>, + F: FnMut(&mut Bencher<'_, M>) + 'static, + { + let routine = NamedRoutine { + id: id.into(), + f: Box::new(RefCell::new(Function::new(move |b, _| f(b)))), + }; + self.routines.push(routine); + self + } + + /// Set the input size for this benchmark group. Used for reporting the + /// throughput. + pub fn throughput(mut self, throughput: Throughput) -> Benchmark<M> { + self.throughput = Some(throughput); + self + } +} + +impl<M: Measurement> BenchmarkDefinition<M> for Benchmark<M> { + fn run(self, group_id: &str, c: &mut Criterion<M>) { + let report_context = ReportContext { + output_directory: c.output_directory.clone(), + plot_config: self.config.plot_config.clone(), + }; + + let config = self.config.to_complete(&c.config); + let num_routines = self.routines.len(); + + let mut all_ids = vec![]; + let mut any_matched = false; + + if let Some(conn) = &c.connection { + conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id }) + .unwrap(); + } + + for routine in self.routines { + let function_id = if num_routines == 1 && group_id == routine.id { + None + } else { + Some(routine.id) + }; + + let mut id = BenchmarkId::new( + group_id.to_owned(), + function_id, + None, + self.throughput.clone(), + ); + + id.ensure_directory_name_unique(&c.all_directories); + c.all_directories.insert(id.as_directory_name().to_owned()); + id.ensure_title_unique(&c.all_titles); + c.all_titles.insert(id.as_title().to_owned()); + + let do_run = c.filter_matches(id.id()); + any_matched |= do_run; + + execute_benchmark( + do_run, + &id, + c, + &config, + &mut *routine.f.borrow_mut(), + &report_context, + &(), + self.throughput.clone(), + ); + + all_ids.push(id); + } + + if let Some(conn) = &c.connection { + conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id }) + .unwrap(); + conn.serve_value_formatter(c.measurement.formatter()) + .unwrap(); + } + + if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() { + c.report + .summarize(&report_context, &all_ids, c.measurement.formatter()); + } + if any_matched { + c.report.group_separator(); + } + } +} + +impl<T, M> ParameterizedBenchmark<T, M> +where + T: Debug + 'static, + M: Measurement + 'static, +{ + benchmark_config!(ParameterizedBenchmark); + + pub(crate) fn with_functions( + functions: Vec<NamedRoutine<T, M>>, + parameters: Vec<T>, + ) -> ParameterizedBenchmark<T, M> { + ParameterizedBenchmark { + config: PartialBenchmarkConfig::default(), + values: parameters, + routines: functions, + throughput: None, + } + } + + /// Create a new parameterized benchmark group and adds the given function + /// to it. + /// The function under test must follow the setup - bench - teardown pattern: + /// + /// # Example + /// + /// ```rust + /// # #[macro_use] extern crate criterion; + /// # use criterion::*; + /// + /// fn bench(c: &mut Criterion) { + /// let parameters = vec![1u64, 2u64, 3u64]; + /// + /// // One-time setup goes here + /// c.bench( + /// "my_group", + /// ParameterizedBenchmark::new( + /// "my_function", + /// |b, param| b.iter(|| { + /// // Code to benchmark using param goes here + /// }), + /// parameters + /// ) + /// ); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + pub fn new<S, F, I>(id: S, f: F, parameters: I) -> ParameterizedBenchmark<T, M> + where + S: Into<String>, + F: FnMut(&mut Bencher<'_, M>, &T) + 'static, + I: IntoIterator<Item = T>, + { + ParameterizedBenchmark { + config: PartialBenchmarkConfig::default(), + values: parameters.into_iter().collect(), + routines: vec![], + throughput: None, + } + .with_function(id, f) + } + + /// Add a function to the benchmark group. + pub fn with_function<S, F>(mut self, id: S, f: F) -> ParameterizedBenchmark<T, M> + where + S: Into<String>, + F: FnMut(&mut Bencher<'_, M>, &T) + 'static, + { + let routine = NamedRoutine { + id: id.into(), + f: Box::new(RefCell::new(Function::new(f))), + }; + self.routines.push(routine); + self + } + + /// Use the given function to calculate the input size for a given input. + pub fn throughput<F>(mut self, throughput: F) -> ParameterizedBenchmark<T, M> + where + F: Fn(&T) -> Throughput + 'static, + { + self.throughput = Some(Box::new(throughput)); + self + } +} +impl<T, M> BenchmarkDefinition<M> for ParameterizedBenchmark<T, M> +where + T: Debug + 'static, + M: Measurement + 'static, +{ + fn run(self, group_id: &str, c: &mut Criterion<M>) { + let report_context = ReportContext { + output_directory: c.output_directory.clone(), + plot_config: self.config.plot_config.clone(), + }; + + let config = self.config.to_complete(&c.config); + let num_parameters = self.values.len(); + let num_routines = self.routines.len(); + + let mut all_ids = vec![]; + let mut any_matched = false; + + if let Some(conn) = &c.connection { + conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id }) + .unwrap(); + } + + for routine in self.routines { + for value in &self.values { + let function_id = if num_routines == 1 && group_id == routine.id { + None + } else { + Some(routine.id.clone()) + }; + + let value_str = if num_parameters == 1 { + None + } else { + Some(format!("{:?}", value)) + }; + + let throughput = self.throughput.as_ref().map(|func| func(value)); + let mut id = BenchmarkId::new( + group_id.to_owned(), + function_id, + value_str, + throughput.clone(), + ); + + id.ensure_directory_name_unique(&c.all_directories); + c.all_directories.insert(id.as_directory_name().to_owned()); + id.ensure_title_unique(&c.all_titles); + c.all_titles.insert(id.as_title().to_owned()); + + let do_run = c.filter_matches(id.id()); + any_matched |= do_run; + + execute_benchmark( + do_run, + &id, + c, + &config, + &mut *routine.f.borrow_mut(), + &report_context, + value, + throughput, + ); + + all_ids.push(id); + } + } + + if let Some(conn) = &c.connection { + conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id }) + .unwrap(); + conn.serve_value_formatter(c.measurement.formatter()) + .unwrap(); + } + + if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() { + c.report + .summarize(&report_context, &all_ids, c.measurement.formatter()); + } + if any_matched { + c.report.group_separator(); + } + } +} + +#[cfg_attr(feature = "cargo-clippy", allow(clippy::too_many_arguments))] +fn execute_benchmark<T, M>( + do_run: bool, + id: &BenchmarkId, + c: &Criterion<M>, + config: &BenchmarkConfig, + routine: &mut dyn Routine<M, T>, + report_context: &ReportContext, + parameter: &T, + throughput: Option<Throughput>, +) where + T: Debug, + M: Measurement, +{ + match c.mode { + Mode::Benchmark => { + if let Some(conn) = &c.connection { + if do_run { + conn.send(&OutgoingMessage::BeginningBenchmark { id: id.into() }) + .unwrap(); + } else { + conn.send(&OutgoingMessage::SkippingBenchmark { id: id.into() }) + .unwrap(); + } + } + + if do_run { + analysis::common( + id, + routine, + config, + c, + report_context, + parameter, + throughput, + ); + } + } + Mode::List => { + if do_run { + println!("{}: bench", id); + } + } + Mode::Test => { + if do_run { + // In test mode, run the benchmark exactly once, then exit. + c.report.test_start(id, report_context); + routine.test(&c.measurement, parameter); + c.report.test_pass(id, report_context); + } + } + Mode::Profile(duration) => { + if do_run { + routine.profile(&c.measurement, id, c, report_context, duration, parameter); + } + } + } +} diff --git a/src/benchmark_group.rs b/src/benchmark_group.rs new file mode 100755 index 0000000..c59cf56 --- /dev/null +++ b/src/benchmark_group.rs @@ -0,0 +1,497 @@ +use crate::analysis; +use crate::benchmark::PartialBenchmarkConfig; +use crate::connection::OutgoingMessage; +use crate::measurement::Measurement; +use crate::report::BenchmarkId as InternalBenchmarkId; +use crate::report::ReportContext; +use crate::routine::{Function, Routine}; +use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput}; +use std::time::Duration; + +/// Structure used to group together a set of related benchmarks, along with custom configuration +/// settings for groups of benchmarks. All benchmarks performed using a benchmark group will be +/// grouped together in the final report. +/// +/// # Examples: +/// +/// ```no_run +/// #[macro_use] extern crate criterion; +/// use self::criterion::*; +/// use std::time::Duration; +/// +/// fn bench_simple(c: &mut Criterion) { +/// let mut group = c.benchmark_group("My Group"); +/// +/// // Now we can perform benchmarks with this group +/// group.bench_function("Bench 1", |b| b.iter(|| 1 )); +/// group.bench_function("Bench 2", |b| b.iter(|| 2 )); +/// +/// // It's recommended to call group.finish() explicitly at the end, but if you don't it will +/// // be called automatically when the group is dropped. +/// group.finish(); +/// } +/// +/// fn bench_nested(c: &mut Criterion) { +/// let mut group = c.benchmark_group("My Second Group"); +/// // We can override the configuration on a per-group level +/// group.measurement_time(Duration::from_secs(1)); +/// +/// // We can also use loops to define multiple benchmarks, even over multiple dimensions. +/// for x in 0..3 { +/// for y in 0..3 { +/// let point = (x, y); +/// let parameter_string = format!("{} * {}", x, y); +/// group.bench_with_input(BenchmarkId::new("Multiply", parameter_string), &point, +/// |b, (p_x, p_y)| b.iter(|| p_x * p_y)); +/// } +/// } +/// +/// group.finish(); +/// } +/// +/// fn bench_throughput(c: &mut Criterion) { +/// let mut group = c.benchmark_group("Summation"); +/// +/// for size in [1024, 2048, 4096].iter() { +/// // Generate input of an appropriate size... +/// let input = vec![1u64, *size]; +/// +/// // We can use the throughput function to tell Criterion.rs how large the input is +/// // so it can calculate the overall throughput of the function. If we wanted, we could +/// // even change the benchmark configuration for different inputs (eg. to reduce the +/// // number of samples for extremely large and slow inputs) or even different functions. +/// group.throughput(Throughput::Elements(*size as u64)); +/// +/// group.bench_with_input(BenchmarkId::new("sum", *size), &input, +/// |b, i| b.iter(|| i.iter().sum::<u64>())); +/// group.bench_with_input(BenchmarkId::new("fold", *size), &input, +/// |b, i| b.iter(|| i.iter().fold(0u64, |a, b| a + b))); +/// } +/// +/// group.finish(); +/// } +/// +/// criterion_group!(benches, bench_simple, bench_nested, bench_throughput); +/// criterion_main!(benches); +/// ``` +pub struct BenchmarkGroup<'a, M: Measurement> { + criterion: &'a mut Criterion<M>, + group_name: String, + all_ids: Vec<InternalBenchmarkId>, + any_matched: bool, + partial_config: PartialBenchmarkConfig, + throughput: Option<Throughput>, +} +impl<'a, M: Measurement> BenchmarkGroup<'a, M> { + /// Changes the size of the sample for this benchmark + /// + /// A bigger sample should yield more accurate results if paired with a sufficiently large + /// measurement time. + /// + /// Sample size must be at least 10. + /// + /// # Panics + /// + /// Panics if n < 10. + pub fn sample_size(&mut self, n: usize) -> &mut Self { + assert!(n >= 10); + + self.partial_config.sample_size = Some(n); + self + } + + /// Changes the warm up time for this benchmark + /// + /// # Panics + /// + /// Panics if the input duration is zero + pub fn warm_up_time(&mut self, dur: Duration) -> &mut Self { + assert!(dur.to_nanos() > 0); + + self.partial_config.warm_up_time = Some(dur); + self + } + + /// Changes the target measurement time for this benchmark group. + /// + /// Criterion will attempt to spent approximately this amount of time measuring each + /// benchmark on a best-effort basis. If it is not possible to perform the measurement in + /// the requested time (eg. because each iteration of the benchmark is long) then Criterion + /// will spend as long as is needed to collect the desired number of samples. With a longer + /// time, the measurement will become more resilient to interference from other programs. + /// + /// # Panics + /// + /// Panics if the input duration is zero + pub fn measurement_time(&mut self, dur: Duration) -> &mut Self { + assert!(dur.to_nanos() > 0); + + self.partial_config.measurement_time = Some(dur); + self + } + + /// Changes the number of resamples for this benchmark group + /// + /// Number of resamples to use for the + /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling) + /// + /// A larger number of resamples reduces the random sampling errors which are inherent to the + /// bootstrap method, but also increases the analysis time. + /// + /// # Panics + /// + /// Panics if the number of resamples is set to zero + pub fn nresamples(&mut self, n: usize) -> &mut Self { + assert!(n > 0); + if n <= 1000 { + println!("\nWarning: It is not recommended to reduce nresamples below 1000."); + } + + self.partial_config.nresamples = Some(n); + self + } + + /// Changes the noise threshold for benchmarks in this group. The noise threshold + /// is used to filter out small changes in performance from one run to the next, even if they + /// are statistically significant. Sometimes benchmarking the same code twice will result in + /// small but statistically significant differences solely because of noise. This provides a way + /// to filter out some of these false positives at the cost of making it harder to detect small + /// changes to the true performance of the benchmark. + /// + /// The default is 0.01, meaning that changes smaller than 1% will be ignored. + /// + /// # Panics + /// + /// Panics if the threshold is set to a negative value + pub fn noise_threshold(&mut self, threshold: f64) -> &mut Self { + assert!(threshold >= 0.0); + + self.partial_config.noise_threshold = Some(threshold); + self + } + + /// Changes the confidence level for benchmarks in this group. The confidence + /// level is the desired probability that the true runtime lies within the estimated + /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is + /// 0.95, meaning that the confidence interval should capture the true value 95% of the time. + /// + /// # Panics + /// + /// Panics if the confidence level is set to a value outside the `(0, 1)` range + pub fn confidence_level(&mut self, cl: f64) -> &mut Self { + assert!(cl > 0.0 && cl < 1.0); + if cl < 0.5 { + println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); + } + + self.partial_config.confidence_level = Some(cl); + self + } + + /// Changes the [significance level](https://en.wikipedia.org/wiki/Statistical_significance) + /// for benchmarks in this group. This is used to perform a + /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if + /// the measurements from this run are different from the measured performance of the last run. + /// The significance level is the desired probability that two measurements of identical code + /// will be considered 'different' due to noise in the measurements. The default value is 0.05, + /// meaning that approximately 5% of identical benchmarks will register as different due to + /// noise. + /// + /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase + /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to + /// detect small but real changes in the performance. By setting the significance level + /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also + /// report more spurious differences. + /// + /// See also the noise threshold setting. + /// + /// # Panics + /// + /// Panics if the significance level is set to a value outside the `(0, 1)` range + pub fn significance_level(&mut self, sl: f64) -> &mut Self { + assert!(sl > 0.0 && sl < 1.0); + + self.partial_config.significance_level = Some(sl); + self + } + + /// Changes the plot configuration for this benchmark group. + pub fn plot_config(&mut self, new_config: PlotConfiguration) -> &mut Self { + self.partial_config.plot_config = new_config; + self + } + + /// Set the input size for this benchmark group. Used for reporting the + /// throughput. + pub fn throughput(&mut self, throughput: Throughput) -> &mut Self { + self.throughput = Some(throughput); + self + } + + /// Set the sampling mode for this benchmark group. + pub fn sampling_mode(&mut self, new_mode: SamplingMode) -> &mut Self { + self.partial_config.sampling_mode = Some(new_mode); + self + } + + pub(crate) fn new(criterion: &mut Criterion<M>, group_name: String) -> BenchmarkGroup<'_, M> { + BenchmarkGroup { + criterion, + group_name, + all_ids: vec![], + any_matched: false, + partial_config: PartialBenchmarkConfig::default(), + throughput: None, + } + } + + /// Benchmark the given parameterless function inside this benchmark group. + pub fn bench_function<ID: IntoBenchmarkId, F>(&mut self, id: ID, mut f: F) -> &mut Self + where + F: FnMut(&mut Bencher<'_, M>), + { + self.run_bench(id.into_benchmark_id(), &(), |b, _| f(b)); + self + } + + /// Benchmark the given parameterized function inside this benchmark group. + pub fn bench_with_input<ID: IntoBenchmarkId, F, I>( + &mut self, + id: ID, + input: &I, + f: F, + ) -> &mut Self + where + F: FnMut(&mut Bencher<'_, M>, &I), + I: ?Sized, + { + self.run_bench(id.into_benchmark_id(), input, f); + self + } + + fn run_bench<F, I>(&mut self, id: BenchmarkId, input: &I, f: F) + where + F: FnMut(&mut Bencher<'_, M>, &I), + I: ?Sized, + { + let config = self.partial_config.to_complete(&self.criterion.config); + let report_context = ReportContext { + output_directory: self.criterion.output_directory.clone(), + plot_config: self.partial_config.plot_config.clone(), + }; + + let mut id = InternalBenchmarkId::new( + self.group_name.clone(), + id.function_name, + id.parameter, + self.throughput.clone(), + ); + + assert!( + !self.all_ids.contains(&id), + "Benchmark IDs must be unique within a group." + ); + + id.ensure_directory_name_unique(&self.criterion.all_directories); + self.criterion + .all_directories + .insert(id.as_directory_name().to_owned()); + id.ensure_title_unique(&self.criterion.all_titles); + self.criterion.all_titles.insert(id.as_title().to_owned()); + + let do_run = self.criterion.filter_matches(id.id()); + self.any_matched |= do_run; + let mut func = Function::new(f); + + match self.criterion.mode { + Mode::Benchmark => { + if let Some(conn) = &self.criterion.connection { + if do_run { + conn.send(&OutgoingMessage::BeginningBenchmark { id: (&id).into() }) + .unwrap(); + } else { + conn.send(&OutgoingMessage::SkippingBenchmark { id: (&id).into() }) + .unwrap(); + } + } + if do_run { + analysis::common( + &id, + &mut func, + &config, + self.criterion, + &report_context, + input, + self.throughput.clone(), + ); + } + } + Mode::List => { + if do_run { + println!("{}: bench", id); + } + } + Mode::Test => { + if do_run { + // In test mode, run the benchmark exactly once, then exit. + self.criterion.report.test_start(&id, &report_context); + func.test(&self.criterion.measurement, input); + self.criterion.report.test_pass(&id, &report_context); + } + } + Mode::Profile(duration) => { + if do_run { + func.profile( + &self.criterion.measurement, + &id, + &self.criterion, + &report_context, + duration, + input, + ); + } + } + } + + self.all_ids.push(id); + } + + /// Consume the benchmark group and generate the summary reports for the group. + /// + /// It is recommended to call this explicitly, but if you forget it will be called when the + /// group is dropped. + pub fn finish(self) { + ::std::mem::drop(self); + } +} +impl<'a, M: Measurement> Drop for BenchmarkGroup<'a, M> { + fn drop(&mut self) { + // I don't really like having a bunch of non-trivial code in drop, but this is the only way + // to really write linear types like this in Rust... + if let Some(conn) = &mut self.criterion.connection { + conn.send(&OutgoingMessage::FinishedBenchmarkGroup { + group: &self.group_name, + }) + .unwrap(); + + conn.serve_value_formatter(self.criterion.measurement.formatter()) + .unwrap(); + } + + if self.all_ids.len() > 1 && self.any_matched && self.criterion.mode.is_benchmark() { + let report_context = ReportContext { + output_directory: self.criterion.output_directory.clone(), + plot_config: self.partial_config.plot_config.clone(), + }; + + self.criterion.report.summarize( + &report_context, + &self.all_ids, + self.criterion.measurement.formatter(), + ); + } + if self.any_matched { + self.criterion.report.group_separator(); + } + } +} + +/// Simple structure representing an ID for a benchmark. The ID must be unique within a benchmark +/// group. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct BenchmarkId { + pub(crate) function_name: Option<String>, + pub(crate) parameter: Option<String>, +} +impl BenchmarkId { + /// Construct a new benchmark ID from a string function name and a parameter value. + /// + /// Note that the parameter value need not be the same as the parameter passed to your + /// actual benchmark. For instance, you might have a benchmark that takes a 1MB string as + /// input. It would be impractical to embed the whole string in the benchmark ID, so instead + /// your parameter value might be a descriptive string like "1MB Alphanumeric". + /// + /// # Examples + /// ``` + /// # use criterion::{BenchmarkId, Criterion}; + /// // A basic benchmark ID is typically constructed from a constant string and a simple + /// // parameter + /// let basic_id = BenchmarkId::new("my_id", 5); + /// + /// // The function name can be a string + /// let function_name = "test_string".to_string(); + /// let string_id = BenchmarkId::new(function_name, 12); + /// + /// // Benchmark IDs are passed to benchmark groups: + /// let mut criterion = Criterion::default(); + /// let mut group = criterion.benchmark_group("My Group"); + /// // Generate a very large input + /// let input : String = ::std::iter::repeat("X").take(1024 * 1024).collect(); + /// + /// // Note that we don't have to use the input as the parameter in the ID + /// group.bench_with_input(BenchmarkId::new("Test long string", "1MB X's"), &input, |b, i| { + /// b.iter(|| i.len()) + /// }); + /// ``` + pub fn new<S: Into<String>, P: ::std::fmt::Display>( + function_name: S, + parameter: P, + ) -> BenchmarkId { + BenchmarkId { + function_name: Some(function_name.into()), + parameter: Some(format!("{}", parameter)), + } + } + + /// Construct a new benchmark ID from just a parameter value. Use this when benchmarking a + /// single function with a variety of different inputs. + pub fn from_parameter<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId { + BenchmarkId { + function_name: None, + parameter: Some(format!("{}", parameter)), + } + } + + pub(crate) fn no_function() -> BenchmarkId { + BenchmarkId { + function_name: None, + parameter: None, + } + } + + pub(crate) fn no_function_with_input<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId { + BenchmarkId { + function_name: None, + parameter: Some(format!("{}", parameter)), + } + } +} + +mod private { + pub trait Sealed {} + impl Sealed for super::BenchmarkId {} + impl<S: Into<String>> Sealed for S {} +} + +/// Sealed trait which allows users to automatically convert strings to benchmark IDs. +pub trait IntoBenchmarkId: private::Sealed { + fn into_benchmark_id(self) -> BenchmarkId; +} +impl IntoBenchmarkId for BenchmarkId { + fn into_benchmark_id(self) -> BenchmarkId { + self + } +} +impl<S: Into<String>> IntoBenchmarkId for S { + fn into_benchmark_id(self) -> BenchmarkId { + let function_name = self.into(); + if function_name.is_empty() { + panic!("Function name must not be empty."); + } + + BenchmarkId { + function_name: Some(function_name), + parameter: None, + } + } +} diff --git a/src/connection.rs b/src/connection.rs new file mode 100755 index 0000000..1f42fff --- /dev/null +++ b/src/connection.rs @@ -0,0 +1,370 @@ +use crate::report::BenchmarkId as InternalBenchmarkId;
+use crate::Throughput;
+use std::cell::RefCell;
+use std::convert::TryFrom;
+use std::io::{Read, Write};
+use std::mem::size_of;
+use std::net::TcpStream;
+
+#[derive(Debug)]
+pub enum MessageError {
+ SerializationError(serde_cbor::Error),
+ IoError(std::io::Error),
+}
+impl From<serde_cbor::Error> for MessageError {
+ fn from(other: serde_cbor::Error) -> Self {
+ MessageError::SerializationError(other)
+ }
+}
+impl From<std::io::Error> for MessageError {
+ fn from(other: std::io::Error) -> Self {
+ MessageError::IoError(other)
+ }
+}
+impl std::fmt::Display for MessageError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ MessageError::SerializationError(error) => write!(
+ f,
+ "Failed to serialize or deserialize message to Criterion.rs benchmark:\n{}",
+ error
+ ),
+ MessageError::IoError(error) => write!(
+ f,
+ "Failed to read or write message to Criterion.rs benchmark:\n{}",
+ error
+ ),
+ }
+ }
+}
+impl std::error::Error for MessageError {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ match self {
+ MessageError::SerializationError(err) => Some(err),
+ MessageError::IoError(err) => Some(err),
+ }
+ }
+}
+
+// Use str::len as a const fn once we bump MSRV over 1.39.
+const RUNNER_MAGIC_NUMBER: &str = "cargo-criterion";
+const RUNNER_HELLO_SIZE: usize = 15 //RUNNER_MAGIC_NUMBER.len() // magic number
+ + (size_of::<u8>() * 3); // version number
+
+const BENCHMARK_MAGIC_NUMBER: &str = "Criterion";
+const BENCHMARK_HELLO_SIZE: usize = 9 //BENCHMARK_MAGIC_NUMBER.len() // magic number
+ + (size_of::<u8>() * 3) // version number
+ + size_of::<u16>() // protocol version
+ + size_of::<u16>(); // protocol format
+const PROTOCOL_VERSION: u16 = 1;
+const PROTOCOL_FORMAT: u16 = 1;
+
+#[derive(Debug)]
+struct InnerConnection {
+ socket: TcpStream,
+ receive_buffer: Vec<u8>,
+ send_buffer: Vec<u8>,
+ runner_version: [u8; 3],
+}
+impl InnerConnection {
+ pub fn new(mut socket: TcpStream) -> Result<Self, std::io::Error> {
+ // read the runner-hello
+ let mut hello_buf = [0u8; RUNNER_HELLO_SIZE];
+ socket.read_exact(&mut hello_buf)?;
+ if &hello_buf[0..RUNNER_MAGIC_NUMBER.len()] != RUNNER_MAGIC_NUMBER.as_bytes() {
+ panic!("Not connected to cargo-criterion.");
+ }
+ let i = RUNNER_MAGIC_NUMBER.len();
+ let runner_version = [hello_buf[i], hello_buf[i + 1], hello_buf[i + 2]];
+
+ info!("Runner version: {:?}", runner_version);
+
+ // now send the benchmark-hello
+ let mut hello_buf = [0u8; BENCHMARK_HELLO_SIZE];
+ hello_buf[0..BENCHMARK_MAGIC_NUMBER.len()]
+ .copy_from_slice(BENCHMARK_MAGIC_NUMBER.as_bytes());
+ let mut i = BENCHMARK_MAGIC_NUMBER.len();
+ hello_buf[i] = env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap();
+ hello_buf[i + 1] = env!("CARGO_PKG_VERSION_MINOR").parse().unwrap();
+ hello_buf[i + 2] = env!("CARGO_PKG_VERSION_PATCH").parse().unwrap();
+ i += 3;
+ hello_buf[i..i + 2].clone_from_slice(&PROTOCOL_VERSION.to_be_bytes());
+ i += 2;
+ hello_buf[i..i + 2].clone_from_slice(&PROTOCOL_FORMAT.to_be_bytes());
+
+ socket.write_all(&hello_buf)?;
+
+ Ok(InnerConnection {
+ socket,
+ receive_buffer: vec![],
+ send_buffer: vec![],
+ runner_version,
+ })
+ }
+
+ #[allow(dead_code)]
+ pub fn recv(&mut self) -> Result<IncomingMessage, MessageError> {
+ let mut length_buf = [0u8; 4];
+ self.socket.read_exact(&mut length_buf)?;
+ let length = u32::from_be_bytes(length_buf);
+ self.receive_buffer.resize(length as usize, 0u8);
+ self.socket.read_exact(&mut self.receive_buffer)?;
+ let value = serde_cbor::from_slice(&self.receive_buffer)?;
+ Ok(value)
+ }
+
+ pub fn send(&mut self, message: &OutgoingMessage) -> Result<(), MessageError> {
+ self.send_buffer.truncate(0);
+ serde_cbor::to_writer(&mut self.send_buffer, message)?;
+ let size = u32::try_from(self.send_buffer.len()).unwrap();
+ let length_buf = size.to_be_bytes();
+ self.socket.write_all(&length_buf)?;
+ self.socket.write_all(&self.send_buffer)?;
+ Ok(())
+ }
+}
+
+/// This is really just a holder to allow us to send messages through a shared reference to the
+/// connection.
+#[derive(Debug)]
+pub struct Connection {
+ inner: RefCell<InnerConnection>,
+}
+impl Connection {
+ pub fn new(socket: TcpStream) -> Result<Self, std::io::Error> {
+ Ok(Connection {
+ inner: RefCell::new(InnerConnection::new(socket)?),
+ })
+ }
+
+ #[allow(dead_code)]
+ pub fn recv(&self) -> Result<IncomingMessage, MessageError> {
+ self.inner.borrow_mut().recv()
+ }
+
+ pub fn send(&self, message: &OutgoingMessage) -> Result<(), MessageError> {
+ self.inner.borrow_mut().send(message)
+ }
+
+ pub fn serve_value_formatter(
+ &self,
+ formatter: &dyn crate::measurement::ValueFormatter,
+ ) -> Result<(), MessageError> {
+ loop {
+ let response = match self.recv()? {
+ IncomingMessage::FormatValue { value } => OutgoingMessage::FormattedValue {
+ value: formatter.format_value(value),
+ },
+ IncomingMessage::FormatThroughput { value, throughput } => {
+ OutgoingMessage::FormattedValue {
+ value: formatter.format_throughput(&throughput, value),
+ }
+ }
+ IncomingMessage::ScaleValues {
+ typical_value,
+ mut values,
+ } => {
+ let unit = formatter.scale_values(typical_value, &mut values);
+ OutgoingMessage::ScaledValues {
+ unit,
+ scaled_values: values,
+ }
+ }
+ IncomingMessage::ScaleThroughputs {
+ typical_value,
+ throughput,
+ mut values,
+ } => {
+ let unit = formatter.scale_throughputs(typical_value, &throughput, &mut values);
+ OutgoingMessage::ScaledValues {
+ unit,
+ scaled_values: values,
+ }
+ }
+ IncomingMessage::ScaleForMachines { mut values } => {
+ let unit = formatter.scale_for_machines(&mut values);
+ OutgoingMessage::ScaledValues {
+ unit,
+ scaled_values: values,
+ }
+ }
+ IncomingMessage::Continue => break,
+ _ => panic!(),
+ };
+ self.send(&response)?;
+ }
+ Ok(())
+ }
+}
+
+/// Enum defining the messages we can receive
+#[derive(Debug, Deserialize)]
+pub enum IncomingMessage {
+ // Value formatter requests
+ FormatValue {
+ value: f64,
+ },
+ FormatThroughput {
+ value: f64,
+ throughput: Throughput,
+ },
+ ScaleValues {
+ typical_value: f64,
+ values: Vec<f64>,
+ },
+ ScaleThroughputs {
+ typical_value: f64,
+ values: Vec<f64>,
+ throughput: Throughput,
+ },
+ ScaleForMachines {
+ values: Vec<f64>,
+ },
+ Continue,
+
+ __Other,
+}
+
+/// Enum defining the messages we can send
+#[derive(Debug, Serialize)]
+pub enum OutgoingMessage<'a> {
+ BeginningBenchmarkGroup {
+ group: &'a str,
+ },
+ FinishedBenchmarkGroup {
+ group: &'a str,
+ },
+ BeginningBenchmark {
+ id: RawBenchmarkId,
+ },
+ SkippingBenchmark {
+ id: RawBenchmarkId,
+ },
+ Warmup {
+ id: RawBenchmarkId,
+ nanos: f64,
+ },
+ MeasurementStart {
+ id: RawBenchmarkId,
+ sample_count: u64,
+ estimate_ns: f64,
+ iter_count: u64,
+ },
+ MeasurementComplete {
+ id: RawBenchmarkId,
+ iters: &'a [f64],
+ times: &'a [f64],
+ plot_config: PlotConfiguration,
+ sampling_method: SamplingMethod,
+ benchmark_config: BenchmarkConfig,
+ },
+ // value formatter responses
+ FormattedValue {
+ value: String,
+ },
+ ScaledValues {
+ scaled_values: Vec<f64>,
+ unit: &'a str,
+ },
+}
+
+// Also define serializable variants of certain things, either to avoid leaking
+// serializability into the public interface or because the serialized form
+// is a bit different from the regular one.
+
+#[derive(Debug, Serialize)]
+pub struct RawBenchmarkId {
+ group_id: String,
+ function_id: Option<String>,
+ value_str: Option<String>,
+ throughput: Vec<Throughput>,
+}
+impl From<&InternalBenchmarkId> for RawBenchmarkId {
+ fn from(other: &InternalBenchmarkId) -> RawBenchmarkId {
+ RawBenchmarkId {
+ group_id: other.group_id.clone(),
+ function_id: other.function_id.clone(),
+ value_str: other.value_str.clone(),
+ throughput: other.throughput.iter().cloned().collect(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub enum AxisScale {
+ Linear,
+ Logarithmic,
+}
+impl From<crate::AxisScale> for AxisScale {
+ fn from(other: crate::AxisScale) -> Self {
+ match other {
+ crate::AxisScale::Linear => AxisScale::Linear,
+ crate::AxisScale::Logarithmic => AxisScale::Logarithmic,
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub struct PlotConfiguration {
+ summary_scale: AxisScale,
+}
+impl From<&crate::PlotConfiguration> for PlotConfiguration {
+ fn from(other: &crate::PlotConfiguration) -> Self {
+ PlotConfiguration {
+ summary_scale: other.summary_scale.into(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+struct Duration {
+ secs: u64,
+ nanos: u32,
+}
+impl From<std::time::Duration> for Duration {
+ fn from(other: std::time::Duration) -> Self {
+ Duration {
+ secs: other.as_secs(),
+ nanos: other.subsec_nanos(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub struct BenchmarkConfig {
+ confidence_level: f64,
+ measurement_time: Duration,
+ noise_threshold: f64,
+ nresamples: usize,
+ sample_size: usize,
+ significance_level: f64,
+ warm_up_time: Duration,
+}
+impl From<&crate::benchmark::BenchmarkConfig> for BenchmarkConfig {
+ fn from(other: &crate::benchmark::BenchmarkConfig) -> Self {
+ BenchmarkConfig {
+ confidence_level: other.confidence_level,
+ measurement_time: other.measurement_time.into(),
+ noise_threshold: other.noise_threshold,
+ nresamples: other.nresamples,
+ sample_size: other.sample_size,
+ significance_level: other.significance_level,
+ warm_up_time: other.warm_up_time.into(),
+ }
+ }
+}
+
+/// Currently not used; defined for forwards compatibility with cargo-criterion.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum SamplingMethod {
+ Linear,
+ Flat,
+}
+impl From<crate::ActualSamplingMode> for SamplingMethod {
+ fn from(other: crate::ActualSamplingMode) -> Self {
+ match other {
+ crate::ActualSamplingMode::Flat => SamplingMethod::Flat,
+ crate::ActualSamplingMode::Linear => SamplingMethod::Linear,
+ }
+ }
+}
diff --git a/src/csv_report.rs b/src/csv_report.rs new file mode 100755 index 0000000..f30b817 --- /dev/null +++ b/src/csv_report.rs @@ -0,0 +1,90 @@ +use crate::error::Result; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, MeasurementData, Report, ReportContext}; +use crate::Throughput; +use csv::Writer; +use std::io::Write; +use std::path::Path; + +#[derive(Serialize)] +struct CsvRow<'a> { + group: &'a str, + function: Option<&'a str>, + value: Option<&'a str>, + throughput_num: Option<&'a str>, + throughput_type: Option<&'a str>, + sample_measured_value: f64, + unit: &'static str, + iteration_count: u64, +} + +struct CsvReportWriter<W: Write> { + writer: Writer<W>, +} +impl<W: Write> CsvReportWriter<W> { + fn write_data( + &mut self, + id: &BenchmarkId, + data: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) -> Result<()> { + let mut data_scaled: Vec<f64> = data.sample_times().as_ref().into(); + let unit = formatter.scale_for_machines(&mut data_scaled); + let group = id.group_id.as_str(); + let function = id.function_id.as_ref().map(String::as_str); + let value = id.value_str.as_ref().map(String::as_str); + let (throughput_num, throughput_type) = match id.throughput { + Some(Throughput::Bytes(bytes)) => (Some(format!("{}", bytes)), Some("bytes")), + Some(Throughput::Elements(elems)) => (Some(format!("{}", elems)), Some("elements")), + None => (None, None), + }; + let throughput_num = throughput_num.as_ref().map(String::as_str); + + for (count, measured_value) in data.iter_counts().iter().zip(data_scaled.into_iter()) { + let row = CsvRow { + group, + function, + value, + throughput_num, + throughput_type, + sample_measured_value: measured_value, + unit, + iteration_count: (*count) as u64, + }; + self.writer.serialize(row)?; + } + Ok(()) + } +} + +pub struct FileCsvReport; +impl FileCsvReport { + fn write_file( + &self, + path: &Path, + id: &BenchmarkId, + measurements: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) -> Result<()> { + let writer = Writer::from_path(path)?; + let mut writer = CsvReportWriter { writer }; + writer.write_data(id, measurements, formatter)?; + Ok(()) + } +} + +impl Report for FileCsvReport { + fn measurement_complete( + &self, + id: &BenchmarkId, + context: &ReportContext, + measurements: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) { + let mut path = context.output_directory.clone(); + path.push(id.as_directory_name()); + path.push("new"); + path.push("raw.csv"); + log_if_err!(self.write_file(&path, id, measurements, formatter)); + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100755 index 0000000..c1ecf76 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,72 @@ +use csv::Error as CsvError; +use serde_json::Error as SerdeError; +use std::error::Error as StdError; +use std::fmt; +use std::io; +use std::path::PathBuf; + +#[derive(Debug)] +pub enum Error { + AccessError { + path: PathBuf, + inner: io::Error, + }, + CopyError { + from: PathBuf, + to: PathBuf, + inner: io::Error, + }, + SerdeError { + path: PathBuf, + inner: SerdeError, + }, + CsvError(CsvError), +} +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::AccessError { path, inner } => { + write!(f, "Failed to access file {:?}: {}", path, inner) + } + Error::CopyError { from, to, inner } => { + write!(f, "Failed to copy file {:?} to {:?}: {}", from, to, inner) + } + Error::SerdeError { path, inner } => write!( + f, + "Failed to read or write file {:?} due to serialization error: {}", + path, inner + ), + Error::CsvError(inner) => write!(f, "CSV error: {}", inner), + } + } +} +impl StdError for Error { + fn description(&self) -> &str { + match self { + Error::AccessError { .. } => "AccessError", + Error::CopyError { .. } => "CopyError", + Error::SerdeError { .. } => "SerdeError", + Error::CsvError(_) => "CsvError", + } + } + + fn cause(&self) -> Option<&dyn StdError> { + match self { + Error::AccessError { inner, .. } => Some(inner), + Error::CopyError { inner, .. } => Some(inner), + Error::SerdeError { inner, .. } => Some(inner), + Error::CsvError(inner) => Some(inner), + } + } +} +impl From<CsvError> for Error { + fn from(other: CsvError) -> Error { + Error::CsvError(other) + } +} + +pub type Result<T> = ::std::result::Result<T, Error>; + +pub(crate) fn log_error(e: &Error) { + error!("error: {}", e); +} diff --git a/src/estimate.rs b/src/estimate.rs new file mode 100755 index 0000000..8a79d27 --- /dev/null +++ b/src/estimate.rs @@ -0,0 +1,184 @@ +use std::fmt; + +use crate::stats::Distribution; + +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize, Debug)] +pub enum Statistic { + Mean, + Median, + MedianAbsDev, + Slope, + StdDev, + Typical, +} + +impl fmt::Display for Statistic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Statistic::Mean => f.pad("mean"), + Statistic::Median => f.pad("median"), + Statistic::MedianAbsDev => f.pad("MAD"), + Statistic::Slope => f.pad("slope"), + Statistic::StdDev => f.pad("SD"), + Statistic::Typical => f.pad("typical"), + } + } +} + +#[derive(Clone, PartialEq, Deserialize, Serialize, Debug)] +pub struct ConfidenceInterval { + pub confidence_level: f64, + pub lower_bound: f64, + pub upper_bound: f64, +} + +#[derive(Clone, PartialEq, Deserialize, Serialize, Debug)] +pub struct Estimate { + /// The confidence interval for this estimate + pub confidence_interval: ConfidenceInterval, + /// + pub point_estimate: f64, + /// The standard error of this estimate + pub standard_error: f64, +} + +pub fn build_estimates( + distributions: &Distributions, + points: &PointEstimates, + cl: f64, +) -> Estimates { + let to_estimate = |point_estimate, distribution: &Distribution<f64>| { + let (lb, ub) = distribution.confidence_interval(cl); + + Estimate { + confidence_interval: ConfidenceInterval { + confidence_level: cl, + lower_bound: lb, + upper_bound: ub, + }, + point_estimate, + standard_error: distribution.std_dev(None), + } + }; + + Estimates { + mean: to_estimate(points.mean, &distributions.mean), + median: to_estimate(points.median, &distributions.median), + median_abs_dev: to_estimate(points.median_abs_dev, &distributions.median_abs_dev), + slope: None, + std_dev: to_estimate(points.std_dev, &distributions.std_dev), + } +} + +pub fn build_change_estimates( + distributions: &ChangeDistributions, + points: &ChangePointEstimates, + cl: f64, +) -> ChangeEstimates { + let to_estimate = |point_estimate, distribution: &Distribution<f64>| { + let (lb, ub) = distribution.confidence_interval(cl); + + Estimate { + confidence_interval: ConfidenceInterval { + confidence_level: cl, + lower_bound: lb, + upper_bound: ub, + }, + point_estimate, + standard_error: distribution.std_dev(None), + } + }; + + ChangeEstimates { + mean: to_estimate(points.mean, &distributions.mean), + median: to_estimate(points.median, &distributions.median), + } +} + +pub struct PointEstimates { + pub mean: f64, + pub median: f64, + pub median_abs_dev: f64, + pub std_dev: f64, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Estimates { + pub mean: Estimate, + pub median: Estimate, + pub median_abs_dev: Estimate, + pub slope: Option<Estimate>, + pub std_dev: Estimate, +} +impl Estimates { + pub fn typical(&self) -> &Estimate { + self.slope.as_ref().unwrap_or(&self.mean) + } + pub fn get(&self, stat: Statistic) -> Option<&Estimate> { + match stat { + Statistic::Mean => Some(&self.mean), + Statistic::Median => Some(&self.median), + Statistic::MedianAbsDev => Some(&self.median_abs_dev), + Statistic::Slope => self.slope.as_ref(), + Statistic::StdDev => Some(&self.std_dev), + Statistic::Typical => Some(self.typical()), + } + } +} + +pub struct Distributions { + pub mean: Distribution<f64>, + pub median: Distribution<f64>, + pub median_abs_dev: Distribution<f64>, + pub slope: Option<Distribution<f64>>, + pub std_dev: Distribution<f64>, +} +impl Distributions { + pub fn typical(&self) -> &Distribution<f64> { + self.slope.as_ref().unwrap_or(&self.mean) + } + pub fn get(&self, stat: Statistic) -> Option<&Distribution<f64>> { + match stat { + Statistic::Mean => Some(&self.mean), + Statistic::Median => Some(&self.median), + Statistic::MedianAbsDev => Some(&self.median_abs_dev), + Statistic::Slope => self.slope.as_ref(), + Statistic::StdDev => Some(&self.std_dev), + Statistic::Typical => Some(self.typical()), + } + } +} + +pub struct ChangePointEstimates { + pub mean: f64, + pub median: f64, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ChangeEstimates { + pub mean: Estimate, + pub median: Estimate, +} +impl ChangeEstimates { + pub fn get(&self, stat: Statistic) -> &Estimate { + match stat { + Statistic::Mean => &self.mean, + Statistic::Median => &self.median, + _ => panic!("Unexpected statistic"), + } + } +} + +pub struct ChangeDistributions { + pub mean: Distribution<f64>, + pub median: Distribution<f64>, +} +impl ChangeDistributions { + pub fn get(&self, stat: Statistic) -> &Distribution<f64> { + match stat { + Statistic::Mean => &self.mean, + Statistic::Median => &self.median, + _ => panic!("Unexpected statistic"), + } + } +} diff --git a/src/format.rs b/src/format.rs new file mode 100755 index 0000000..984f0f5 --- /dev/null +++ b/src/format.rs @@ -0,0 +1,104 @@ +pub fn change(pct: f64, signed: bool) -> String { + if signed { + format!("{:>+6}%", signed_short(pct * 1e2)) + } else { + format!("{:>6}%", short(pct * 1e2)) + } +} + +pub fn time(ns: f64) -> String { + if ns < 1.0 { + format!("{:>6} ps", short(ns * 1e3)) + } else if ns < 10f64.powi(3) { + format!("{:>6} ns", short(ns)) + } else if ns < 10f64.powi(6) { + format!("{:>6} us", short(ns / 1e3)) + } else if ns < 10f64.powi(9) { + format!("{:>6} ms", short(ns / 1e6)) + } else { + format!("{:>6} s", short(ns / 1e9)) + } +} + +pub fn short(n: f64) -> String { + if n < 10.0 { + format!("{:.4}", n) + } else if n < 100.0 { + format!("{:.3}", n) + } else if n < 1000.0 { + format!("{:.2}", n) + } else if n < 10000.0 { + format!("{:.1}", n) + } else { + format!("{:.0}", n) + } +} + +fn signed_short(n: f64) -> String { + let n_abs = n.abs(); + + if n_abs < 10.0 { + format!("{:+.4}", n) + } else if n_abs < 100.0 { + format!("{:+.3}", n) + } else if n_abs < 1000.0 { + format!("{:+.2}", n) + } else if n_abs < 10000.0 { + format!("{:+.1}", n) + } else { + format!("{:+.0}", n) + } +} + +pub fn iter_count(iterations: u64) -> String { + if iterations < 10_000 { + format!("{} iterations", iterations) + } else if iterations < 1_000_000 { + format!("{:.0}k iterations", (iterations as f64) / 1000.0) + } else if iterations < 10_000_000 { + format!("{:.1}M iterations", (iterations as f64) / (1000.0 * 1000.0)) + } else if iterations < 1_000_000_000 { + format!("{:.0}M iterations", (iterations as f64) / (1000.0 * 1000.0)) + } else if iterations < 10_000_000_000 { + format!( + "{:.1}B iterations", + (iterations as f64) / (1000.0 * 1000.0 * 1000.0) + ) + } else { + format!( + "{:.0}B iterations", + (iterations as f64) / (1000.0 * 1000.0 * 1000.0) + ) + } +} + +pub fn integer(n: f64) -> String { + format!("{}", n as u64) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn short_max_len() { + let mut float = 1.0; + while float < 999_999.9 { + let string = short(float); + println!("{}", string); + assert!(string.len() <= 6); + float *= 2.0; + } + } + + #[test] + fn signed_short_max_len() { + let mut float = -1.0; + while float > -999_999.9 { + let string = signed_short(float); + println!("{}", string); + assert!(string.len() <= 7); + float *= 2.0; + } + } +} diff --git a/src/fs.rs b/src/fs.rs new file mode 100755 index 0000000..f47508b --- /dev/null +++ b/src/fs.rs @@ -0,0 +1,112 @@ +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io::Read; +use std::path::Path; +use walkdir::{DirEntry, WalkDir}; + +use crate::error::{Error, Result}; +use crate::report::BenchmarkId; + +pub fn load<A, P: ?Sized>(path: &P) -> Result<A> +where + A: DeserializeOwned, + P: AsRef<Path>, +{ + let path = path.as_ref(); + let mut f = File::open(path).map_err(|inner| Error::AccessError { + inner, + path: path.to_owned(), + })?; + let mut string = String::new(); + let _ = f.read_to_string(&mut string); + let result: A = serde_json::from_str(string.as_str()).map_err(|inner| Error::SerdeError { + inner, + path: path.to_owned(), + })?; + + Ok(result) +} + +pub fn is_dir<P>(path: &P) -> bool +where + P: AsRef<Path>, +{ + let path: &Path = path.as_ref(); + path.is_dir() +} + +pub fn mkdirp<P>(path: &P) -> Result<()> +where + P: AsRef<Path>, +{ + fs::create_dir_all(path.as_ref()).map_err(|inner| Error::AccessError { + inner, + path: path.as_ref().to_owned(), + })?; + Ok(()) +} + +pub fn cp(from: &Path, to: &Path) -> Result<()> { + fs::copy(from, to).map_err(|inner| Error::CopyError { + inner, + from: from.to_owned(), + to: to.to_owned(), + })?; + Ok(()) +} + +pub fn save<D, P>(data: &D, path: &P) -> Result<()> +where + D: Serialize, + P: AsRef<Path>, +{ + let buf = serde_json::to_string(&data).map_err(|inner| Error::SerdeError { + path: path.as_ref().to_owned(), + inner, + })?; + save_string(&buf, path) +} + +pub fn save_string<P>(data: &str, path: &P) -> Result<()> +where + P: AsRef<Path>, +{ + use std::io::Write; + + File::create(path) + .and_then(|mut f| f.write_all(data.as_bytes())) + .map_err(|inner| Error::AccessError { + inner, + path: path.as_ref().to_owned(), + })?; + + Ok(()) +} + +pub fn list_existing_benchmarks<P>(directory: &P) -> Result<Vec<BenchmarkId>> +where + P: AsRef<Path>, +{ + fn is_benchmark(entry: &DirEntry) -> bool { + // Look for benchmark.json files inside folders named "new" (because we want to ignore + // the baselines) + entry.file_name() == OsStr::new("benchmark.json") + && entry.path().parent().unwrap().file_name().unwrap() == OsStr::new("new") + } + + let mut ids = vec![]; + + for entry in WalkDir::new(directory) + .into_iter() + // Ignore errors. + .filter_map(::std::result::Result::ok) + .filter(is_benchmark) + { + let id: BenchmarkId = load(entry.path())?; + ids.push(id); + } + + Ok(ids) +} diff --git a/src/html/benchmark_report.html.tt b/src/html/benchmark_report.html.tt new file mode 100755 index 0000000..5b0679e --- /dev/null +++ b/src/html/benchmark_report.html.tt @@ -0,0 +1,308 @@ +<!DOCTYPE html> +<html> + +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <title>{title} - Criterion.rs</title> + <style type="text/css"> + body \{ + font: 14px Helvetica Neue; + text-rendering: optimizelegibility; + } + + .body \{ + width: 960px; + margin: auto; + } + + th \{ + font-weight: 200 + } + + th, + td \{ + padding-right: 3px; + padding-bottom: 3px; + } + + a:link \{ + color: #1F78B4; + text-decoration: none; + } + + th.ci-bound \{ + opacity: 0.6 + } + + td.ci-bound \{ + opacity: 0.5 + } + + .stats \{ + width: 80%; + margin: auto; + display: flex; + } + + .additional_stats \{ + flex: 0 0 60% + } + + .additional_plots \{ + flex: 1 + } + + h2 \{ + font-size: 36px; + font-weight: 300; + } + + h3 \{ + font-size: 24px; + font-weight: 300; + } + + #footer \{ + height: 40px; + background: #888; + color: white; + font-size: larger; + font-weight: 300; + } + + #footer a \{ + color: white; + text-decoration: underline; + } + + #footer p \{ + text-align: center + } + </style> +</head> + +<body> + <div class="body"> + <h2>{title}</h2> + <div class="absolute"> + <section class="plots"> + <table width="100%"> + <tbody> + <tr> + <td> + <a href="pdf.svg"> + <img src="pdf_small.svg" alt="PDF of Slope" width="{thumbnail_width}" height="{thumbnail_height}" /> + </a> + </td> + <td> + {{- if slope }} + <a href="regression.svg"> + <img src="regression_small.svg" alt="Regression" width="{thumbnail_width}" height="{thumbnail_height}" /> + </a> + {{- else }} + <a href="iteration_times.svg"> + <img src="iteration_times_small.svg" alt="Iteration Times" width="{thumbnail_width}" height="{thumbnail_height}" /> + </a> + {{- endif }} + </td> + </tr> + </tbody> + </table> + </section> + <section class="stats"> + <div class="additional_stats"> + <h4>Additional Statistics:</h4> + <table> + <thead> + <tr> + <th></th> + <th title="{confidence} confidence level" class="ci-bound">Lower bound</th> + <th>Estimate</th> + <th title="{confidence} confidence level" class="ci-bound">Upper bound</th> + </tr> + </thead> + <tbody> + {{- if slope }} + <tr> + <td>Slope</td> + <td class="ci-bound">{slope.lower}</td> + <td>{slope.point}</td> + <td class="ci-bound">{slope.upper}</td> + </tr> + {{- endif }} + {{- if throughput }} + <tr> + <td>Throughput</td> + <td class="ci-bound">{throughput.lower}</td> + <td>{throughput.point}</td> + <td class="ci-bound">{throughput.upper}</td> + </tr> + {{- endif }} + <tr> + <td>R²</td> + <td class="ci-bound">{r2.lower}</td> + <td>{r2.point}</td> + <td class="ci-bound">{r2.upper}</td> + </tr> + <tr> + <td>Mean</td> + <td class="ci-bound">{mean.lower}</td> + <td>{mean.point}</td> + <td class="ci-bound">{mean.upper}</td> + </tr> + <tr> + <td title="Standard Deviation">Std. Dev.</td> + <td class="ci-bound">{std_dev.lower}</td> + <td>{std_dev.point}</td> + <td class="ci-bound">{std_dev.upper}</td> + </tr> + <tr> + <td>Median</td> + <td class="ci-bound">{median.lower}</td> + <td>{median.point}</td> + <td class="ci-bound">{median.upper}</td> + </tr> + <tr> + <td title="Median Absolute Deviation">MAD</td> + <td class="ci-bound">{mad.lower}</td> + <td>{mad.point}</td> + <td class="ci-bound">{mad.upper}</td> + </tr> + </tbody> + </table> + </div> + {{- if additional_plots }} + <div class="additional_plots"> + <h4>Additional Plots:</h4> + <ul> + {{for plot in additional_plots }} + <li> + <a href="{plot.url}">{plot.name}</a> + </li> + {{- endfor }} + </ul> + </div> + {{- endif }} + </section> + <section class="explanation"> + <h4>Understanding this report:</h4> + <p>The plot on the left displays the average time per iteration for this benchmark. The shaded region + shows the estimated probabilty of an iteration taking a certain amount of time, while the line + shows the mean. Click on the plot for a larger view showing the outliers.</p> + {{- if slope }} + <p>The plot on the right shows the linear regression calculated from the measurements. Each point + represents a sample, though here it shows the total time for the sample rather than time per + iteration. The line is the line of best fit for these measurements.</p> + {{- else }} + <p>The plot on the right shows the average time per iteration for the samples. Each point + represents one sample.</p> + {{- endif }} + <p>See <a href="https://bheisler.github.io/criterion.rs/book/user_guide/command_line_output.html#additional-statistics">the + documentation</a> for more details on the additional statistics.</p> + </section> + </div> + {{- if comparison }} + <section class="plots"> + <h3>Change Since Previous Benchmark</h3> + <div class="relative"> + <table width="100%"> + <tbody> + <tr> + <td> + <a href="both/pdf.svg"> + <img src="relative_pdf_small.svg" alt="PDF Comparison" width="{thumbnail_width}" + height="{thumbnail_height}" /> + </a> + </td> + <td> + {{- if slope }} + <a href="both/regression.svg"> + <img src="relative_regression_small.svg" alt="Regression Comparison" width="{thumbnail_width}" + height="{thumbnail_height}" /> + </a> + {{- else }} + <a href="both/iteration_times.svg"> + <img src="relative_iteration_times_small.svg" alt="Iteration Time Comparison" width="{thumbnail_width}" + height="{thumbnail_height}" /> + </a> + {{- endif }} + </td> + </tr> + </tbody> + </table> + </div> + </section> + <section class="stats"> + <div class="additional_stats"> + <h4>Additional Statistics:</h4> + <table> + <thead> + <tr> + <th></th> + <th title="{confidence} confidence level" class="ci-bound">Lower bound</th> + <th>Estimate</th> + <th title="{confidence} confidence level" class="ci-bound">Upper bound</th> + <th></th> + </tr> + </thead> + <tbody> + <tr> + <td>Change in time</td> + <td class="ci-bound">{comparison.change.lower}</td> + <td>{comparison.change.point}</td> + <td class="ci-bound">{comparison.change.upper}</td> + <td>(p = {comparison.p_value} {comparison.inequality} + {comparison.significance_level})</td> + </tr> + {{- if comparison.thrpt_change }} + <tr> + <td>Change in throughput</td> + <td class="ci-bound">{comparison.thrpt_change.lower}</td> + <td>{comparison.thrpt_change.point}</td> + <td class="ci-bound">{comparison.thrpt_change.upper}</td> + <td></td> + </tr> + {{- endif }} + </tbody> + </table> + {comparison.explanation} + </div> + {{- if comparison.additional_plots }} + <div class="additional_plots"> + <h4>Additional Plots:</h4> + <ul> + {{ for plot in comparison.additional_plots }} + <li> + <a href="{plot.url}">{plot.name}</a> + </li> + {{- endfor }} + </ul> + </div> + {{- endif }} + </section> + <section class="explanation"> + <h4>Understanding this report:</h4> + <p>The plot on the left shows the probability of the function taking a certain amount of time. The red + curve represents the saved measurements from the last time this benchmark was run, while the blue curve + shows the measurements from this run. The lines represent the mean time per iteration. Click on the + plot for a larger view.</p> + {{- if slope }} + <p>The plot on the right shows the two regressions. Again, the red line represents the previous measurement + while the blue line shows the current measurement.</p> + {{- else }} + <p>The plot on the right shows the iteration times for the two measurements. Again, the red dots represent + the previous measurement while the blue dots show the current measurement.</p> + {{- endif}} + <p>See <a href="https://bheisler.github.io/criterion.rs/book/user_guide/command_line_output.html#change">the + documentation</a> for more details on the additional statistics.</p> + </section> + {{- endif }} + </div> + <div id="footer"> + <p>This report was generated by + <a href="https://github.com/bheisler/criterion.rs">Criterion.rs</a>, a statistics-driven benchmarking + library in Rust.</p> + </div> +</body> + +</html>
\ No newline at end of file diff --git a/src/html/index.html.tt b/src/html/index.html.tt new file mode 100755 index 0000000..7c307ed --- /dev/null +++ b/src/html/index.html.tt @@ -0,0 +1,119 @@ +<!DOCTYPE html> +<html> + +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <title>Index - Criterion.rs</title> + <style type="text/css"> + body \{ + font: 14px Helvetica Neue; + text-rendering: optimizelegibility; + } + + .body \{ + width: 960px; + margin: auto; + } + + a:link \{ + color: #1F78B4; + text-decoration: none; + } + + h2 \{ + font-size: 36px; + font-weight: 300; + } + + h3 \{ + font-size: 24px; + font-weight: 300; + } + + #footer \{ + height: 40px; + background: #888; + color: white; + font-size: larger; + font-weight: 300; + } + + #footer a \{ + color: white; + text-decoration: underline; + } + + #footer p \{ + text-align: center + } + + table \{ + border-collapse: collapse; + } + + table, + th, + td \{ + border: 1px solid #888; + } + </style> +</head> + +<body> + <div class="body"> + <h2>Criterion.rs Benchmark Index</h2> + See individual benchmark pages below for more details. + <ul> + {{- for group in groups }} + <li>{{ call report_link with group.group_report }}</li> + {{- if group.function_ids }} + {{- if group.values }} + {# Function ids and values #} + <ul> + <li> + <table> + <tr> + <th></th> + {{- for func in group.function_ids }} + <th>{{ call report_link with func }}</th> + {{- endfor }} + </tr> + {{- for row in group.individual_links }} + <tr> + <th>{{ call report_link with row.value }}</th> + {{- for bench in row.benchmarks }} + <td>{{ call report_link with bench }}</td> + {{- endfor }} + </tr> + {{- endfor }} + </table> + </li> + </ul> + {{- else }} + {# Function IDs but not values #} + <ul> + {{- for func in group.function_ids }} + <li>{{ call report_link with func }}</li> + {{- endfor }} + </ul> + {{- endif }} + {{- else }} + {{- if group.values }} + {# Values but not function ids #} + <ul> + {{- for val in group.values }} + <li>{{ call report_link with val }}</li> + {{- endfor }} + </ul> + {{- endif }} + {{- endif }} + {{- endfor }} + </ul> + </div> + <div id="footer"> + <p>This report was generated by + <a href="https://github.com/bheisler/criterion.rs">Criterion.rs</a>, a statistics-driven benchmarking + library in Rust.</p> + </div> +</body> +</html>
\ No newline at end of file diff --git a/src/html/mod.rs b/src/html/mod.rs new file mode 100755 index 0000000..02ff0de --- /dev/null +++ b/src/html/mod.rs @@ -0,0 +1,837 @@ +use crate::report::{make_filename_safe, BenchmarkId, MeasurementData, Report, ReportContext}; +use crate::stats::bivariate::regression::Slope; + +use crate::estimate::Estimate; +use crate::format; +use crate::fs; +use crate::measurement::ValueFormatter; +use crate::plot::{PlotContext, PlotData, Plotter}; +use crate::SavedSample; +use criterion_plot::Size; +use serde::Serialize; +use std::cell::RefCell; +use std::cmp::Ordering; +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use tinytemplate::TinyTemplate; + +const THUMBNAIL_SIZE: Option<Size> = Some(Size(450, 300)); + +fn debug_context<S: Serialize>(path: &Path, context: &S) { + if crate::debug_enabled() { + let mut context_path = PathBuf::from(path); + context_path.set_extension("json"); + println!("Writing report context to {:?}", context_path); + let result = fs::save(context, &context_path); + if let Err(e) = result { + error!("Failed to write report context debug output: {}", e); + } + } +} + +#[derive(Serialize)] +struct Context { + title: String, + confidence: String, + + thumbnail_width: usize, + thumbnail_height: usize, + + slope: Option<ConfidenceInterval>, + r2: ConfidenceInterval, + mean: ConfidenceInterval, + std_dev: ConfidenceInterval, + median: ConfidenceInterval, + mad: ConfidenceInterval, + throughput: Option<ConfidenceInterval>, + + additional_plots: Vec<Plot>, + + comparison: Option<Comparison>, +} + +#[derive(Serialize)] +struct IndividualBenchmark { + name: String, + path: String, + regression_exists: bool, +} +impl IndividualBenchmark { + fn from_id( + output_directory: &Path, + path_prefix: &str, + id: &BenchmarkId, + ) -> IndividualBenchmark { + let mut regression_path = PathBuf::from(output_directory); + regression_path.push(id.as_directory_name()); + regression_path.push("report"); + regression_path.push("regression.svg"); + + IndividualBenchmark { + name: id.as_title().to_owned(), + path: format!("{}/{}", path_prefix, id.as_directory_name()), + regression_exists: regression_path.is_file(), + } + } +} + +#[derive(Serialize)] +struct SummaryContext { + group_id: String, + + thumbnail_width: usize, + thumbnail_height: usize, + + violin_plot: Option<String>, + line_chart: Option<String>, + + benchmarks: Vec<IndividualBenchmark>, +} + +#[derive(Serialize)] +struct ConfidenceInterval { + lower: String, + upper: String, + point: String, +} + +#[derive(Serialize)] +struct Plot { + name: String, + url: String, +} +impl Plot { + fn new(name: &str, url: &str) -> Plot { + Plot { + name: name.to_owned(), + url: url.to_owned(), + } + } +} + +#[derive(Serialize)] +struct Comparison { + p_value: String, + inequality: String, + significance_level: String, + explanation: String, + + change: ConfidenceInterval, + thrpt_change: Option<ConfidenceInterval>, + additional_plots: Vec<Plot>, +} + +fn if_exists(output_directory: &Path, path: &Path) -> Option<String> { + let report_path = path.join("report/index.html"); + if PathBuf::from(output_directory).join(&report_path).is_file() { + Some(report_path.to_string_lossy().to_string()) + } else { + None + } +} +#[derive(Serialize, Debug)] +struct ReportLink<'a> { + name: &'a str, + path: Option<String>, +} +impl<'a> ReportLink<'a> { + // TODO: Would be nice if I didn't have to keep making these components filename-safe. + fn group(output_directory: &Path, group_id: &'a str) -> ReportLink<'a> { + let path = PathBuf::from(make_filename_safe(group_id)); + + ReportLink { + name: group_id, + path: if_exists(output_directory, &path), + } + } + + fn function(output_directory: &Path, group_id: &str, function_id: &'a str) -> ReportLink<'a> { + let mut path = PathBuf::from(make_filename_safe(group_id)); + path.push(make_filename_safe(function_id)); + + ReportLink { + name: function_id, + path: if_exists(output_directory, &path), + } + } + + fn value(output_directory: &Path, group_id: &str, value_str: &'a str) -> ReportLink<'a> { + let mut path = PathBuf::from(make_filename_safe(group_id)); + path.push(make_filename_safe(value_str)); + + ReportLink { + name: value_str, + path: if_exists(output_directory, &path), + } + } + + fn individual(output_directory: &Path, id: &'a BenchmarkId) -> ReportLink<'a> { + let path = PathBuf::from(id.as_directory_name()); + ReportLink { + name: id.as_title(), + path: if_exists(output_directory, &path), + } + } +} + +#[derive(Serialize)] +struct BenchmarkValueGroup<'a> { + value: Option<ReportLink<'a>>, + benchmarks: Vec<ReportLink<'a>>, +} + +#[derive(Serialize)] +struct BenchmarkGroup<'a> { + group_report: ReportLink<'a>, + + function_ids: Option<Vec<ReportLink<'a>>>, + values: Option<Vec<ReportLink<'a>>>, + + individual_links: Vec<BenchmarkValueGroup<'a>>, +} +impl<'a> BenchmarkGroup<'a> { + fn new(output_directory: &Path, ids: &[&'a BenchmarkId]) -> BenchmarkGroup<'a> { + let group_id = &ids[0].group_id; + let group_report = ReportLink::group(output_directory, group_id); + + let mut function_ids = Vec::with_capacity(ids.len()); + let mut values = Vec::with_capacity(ids.len()); + let mut individual_links = HashMap::with_capacity(ids.len()); + + for id in ids.iter() { + let function_id = id.function_id.as_ref().map(String::as_str); + let value = id.value_str.as_ref().map(String::as_str); + + let individual_link = ReportLink::individual(output_directory, id); + + function_ids.push(function_id); + values.push(value); + + individual_links.insert((function_id, value), individual_link); + } + + fn parse_opt(os: &Option<&str>) -> Option<f64> { + os.and_then(|s| s.parse::<f64>().ok()) + } + + // If all of the value strings can be parsed into a number, sort/dedupe + // numerically. Otherwise sort lexicographically. + if values.iter().all(|os| parse_opt(os).is_some()) { + values.sort_unstable_by(|v1, v2| { + let num1 = parse_opt(&v1); + let num2 = parse_opt(&v2); + + num1.partial_cmp(&num2).unwrap_or(Ordering::Less) + }); + values.dedup_by_key(|os| parse_opt(&os).unwrap()); + } else { + values.sort_unstable(); + values.dedup(); + } + + // Sort and dedupe functions by name. + function_ids.sort_unstable(); + function_ids.dedup(); + + let mut value_groups = Vec::with_capacity(values.len()); + for value in values.iter() { + let row = function_ids + .iter() + .filter_map(|f| individual_links.remove(&(*f, *value))) + .collect::<Vec<_>>(); + value_groups.push(BenchmarkValueGroup { + value: value.map(|s| ReportLink::value(output_directory, group_id, s)), + benchmarks: row, + }); + } + + let function_ids = function_ids + .into_iter() + .map(|os| os.map(|s| ReportLink::function(output_directory, group_id, s))) + .collect::<Option<Vec<_>>>(); + let values = values + .into_iter() + .map(|os| os.map(|s| ReportLink::value(output_directory, group_id, s))) + .collect::<Option<Vec<_>>>(); + + BenchmarkGroup { + group_report, + function_ids, + values, + individual_links: value_groups, + } + } +} + +#[derive(Serialize)] +struct IndexContext<'a> { + groups: Vec<BenchmarkGroup<'a>>, +} + +pub struct Html { + templates: TinyTemplate<'static>, + plotter: RefCell<Box<dyn Plotter>>, +} +impl Html { + pub(crate) fn new(plotter: Box<dyn Plotter>) -> Html { + let mut templates = TinyTemplate::new(); + templates + .add_template("report_link", include_str!("report_link.html.tt")) + .expect("Unable to parse report_link template."); + templates + .add_template("index", include_str!("index.html.tt")) + .expect("Unable to parse index template."); + templates + .add_template("benchmark_report", include_str!("benchmark_report.html.tt")) + .expect("Unable to parse benchmark_report template"); + templates + .add_template("summary_report", include_str!("summary_report.html.tt")) + .expect("Unable to parse summary_report template"); + + let plotter = RefCell::new(plotter); + Html { templates, plotter } + } +} +impl Report for Html { + fn measurement_complete( + &self, + id: &BenchmarkId, + report_context: &ReportContext, + measurements: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) { + try_else_return!({ + let mut report_dir = report_context.output_directory.clone(); + report_dir.push(id.as_directory_name()); + report_dir.push("report"); + fs::mkdirp(&report_dir) + }); + + let typical_estimate = &measurements.absolute_estimates.typical(); + + let time_interval = |est: &Estimate| -> ConfidenceInterval { + ConfidenceInterval { + lower: formatter.format_value(est.confidence_interval.lower_bound), + point: formatter.format_value(est.point_estimate), + upper: formatter.format_value(est.confidence_interval.upper_bound), + } + }; + + let data = measurements.data; + + elapsed! { + "Generating plots", + self.generate_plots(id, report_context, formatter, measurements) + } + + let mut additional_plots = vec![ + Plot::new("Typical", "typical.svg"), + Plot::new("Mean", "mean.svg"), + Plot::new("Std. Dev.", "SD.svg"), + Plot::new("Median", "median.svg"), + Plot::new("MAD", "MAD.svg"), + ]; + if measurements.absolute_estimates.slope.is_some() { + additional_plots.push(Plot::new("Slope", "slope.svg")); + } + + let throughput = measurements + .throughput + .as_ref() + .map(|thr| ConfidenceInterval { + lower: formatter + .format_throughput(thr, typical_estimate.confidence_interval.upper_bound), + upper: formatter + .format_throughput(thr, typical_estimate.confidence_interval.lower_bound), + point: formatter.format_throughput(thr, typical_estimate.point_estimate), + }); + + let context = Context { + title: id.as_title().to_owned(), + confidence: format!( + "{:.2}", + typical_estimate.confidence_interval.confidence_level + ), + + thumbnail_width: THUMBNAIL_SIZE.unwrap().0, + thumbnail_height: THUMBNAIL_SIZE.unwrap().1, + + slope: measurements + .absolute_estimates + .slope + .as_ref() + .map(time_interval), + mean: time_interval(&measurements.absolute_estimates.mean), + median: time_interval(&measurements.absolute_estimates.median), + mad: time_interval(&measurements.absolute_estimates.median_abs_dev), + std_dev: time_interval(&measurements.absolute_estimates.std_dev), + throughput, + + r2: ConfidenceInterval { + lower: format!( + "{:0.7}", + Slope(typical_estimate.confidence_interval.lower_bound).r_squared(&data) + ), + upper: format!( + "{:0.7}", + Slope(typical_estimate.confidence_interval.upper_bound).r_squared(&data) + ), + point: format!( + "{:0.7}", + Slope(typical_estimate.point_estimate).r_squared(&data) + ), + }, + + additional_plots, + + comparison: self.comparison(measurements), + }; + + let mut report_path = report_context.output_directory.clone(); + report_path.push(id.as_directory_name()); + report_path.push("report"); + report_path.push("index.html"); + debug_context(&report_path, &context); + + let text = self + .templates + .render("benchmark_report", &context) + .expect("Failed to render benchmark report template"); + try_else_return!(fs::save_string(&text, &report_path)); + } + + fn summarize( + &self, + context: &ReportContext, + all_ids: &[BenchmarkId], + formatter: &dyn ValueFormatter, + ) { + let all_ids = all_ids + .iter() + .filter(|id| { + let id_dir = context.output_directory.join(id.as_directory_name()); + fs::is_dir(&id_dir) + }) + .collect::<Vec<_>>(); + if all_ids.is_empty() { + return; + } + + let group_id = all_ids[0].group_id.clone(); + + let data = self.load_summary_data(&context.output_directory, &all_ids); + + let mut function_ids = BTreeSet::new(); + let mut value_strs = Vec::with_capacity(all_ids.len()); + for id in all_ids { + if let Some(ref function_id) = id.function_id { + function_ids.insert(function_id); + } + if let Some(ref value_str) = id.value_str { + value_strs.push(value_str); + } + } + + fn try_parse(s: &str) -> Option<f64> { + s.parse::<f64>().ok() + } + + // If all of the value strings can be parsed into a number, sort/dedupe + // numerically. Otherwise sort lexicographically. + if value_strs.iter().all(|os| try_parse(&*os).is_some()) { + value_strs.sort_unstable_by(|v1, v2| { + let num1 = try_parse(&v1); + let num2 = try_parse(&v2); + + num1.partial_cmp(&num2).unwrap_or(Ordering::Less) + }); + value_strs.dedup_by_key(|os| try_parse(&os).unwrap()); + } else { + value_strs.sort_unstable(); + value_strs.dedup(); + } + + for function_id in function_ids { + let samples_with_function: Vec<_> = data + .iter() + .by_ref() + .filter(|&&(ref id, _)| id.function_id.as_ref() == Some(&function_id)) + .collect(); + + if samples_with_function.len() > 1 { + let subgroup_id = + BenchmarkId::new(group_id.clone(), Some(function_id.clone()), None, None); + + self.generate_summary( + &subgroup_id, + &*samples_with_function, + context, + formatter, + false, + ); + } + } + + for value_str in value_strs { + let samples_with_value: Vec<_> = data + .iter() + .by_ref() + .filter(|&&(ref id, _)| id.value_str.as_ref() == Some(&value_str)) + .collect(); + + if samples_with_value.len() > 1 { + let subgroup_id = + BenchmarkId::new(group_id.clone(), None, Some(value_str.clone()), None); + + self.generate_summary( + &subgroup_id, + &*samples_with_value, + context, + formatter, + false, + ); + } + } + + let mut all_data = data.iter().by_ref().collect::<Vec<_>>(); + // First sort the ids/data by value. + // If all of the value strings can be parsed into a number, sort/dedupe + // numerically. Otherwise sort lexicographically. + let all_values_numeric = all_data.iter().all(|(ref id, _)| { + id.value_str + .as_ref() + .map(String::as_str) + .and_then(try_parse) + .is_some() + }); + if all_values_numeric { + all_data.sort_unstable_by(|(a, _), (b, _)| { + let num1 = a.value_str.as_ref().map(String::as_str).and_then(try_parse); + let num2 = b.value_str.as_ref().map(String::as_str).and_then(try_parse); + + num1.partial_cmp(&num2).unwrap_or(Ordering::Less) + }); + } else { + all_data.sort_unstable_by_key(|(id, _)| id.value_str.as_ref()); + } + // Next, sort the ids/data by function name. This results in a sorting priority of + // function name, then value. This one has to be a stable sort. + all_data.sort_by_key(|(id, _)| id.function_id.as_ref()); + + self.generate_summary( + &BenchmarkId::new(group_id, None, None, None), + &*(all_data), + context, + formatter, + true, + ); + self.plotter.borrow_mut().wait(); + } + + fn final_summary(&self, report_context: &ReportContext) { + let output_directory = &report_context.output_directory; + if !fs::is_dir(&output_directory) { + return; + } + + let mut found_ids = try_else_return!(fs::list_existing_benchmarks(&output_directory)); + found_ids.sort_unstable_by_key(|id| id.id().to_owned()); + + // Group IDs by group id + let mut id_groups: HashMap<&str, Vec<&BenchmarkId>> = HashMap::new(); + for id in found_ids.iter() { + id_groups + .entry(&id.group_id) + .or_insert_with(Vec::new) + .push(id); + } + + let mut groups = id_groups + .into_iter() + .map(|(_, group)| BenchmarkGroup::new(output_directory, &group)) + .collect::<Vec<BenchmarkGroup<'_>>>(); + groups.sort_unstable_by_key(|g| g.group_report.name); + + try_else_return!(fs::mkdirp(&output_directory.join("report"))); + + let report_path = output_directory.join("report").join("index.html"); + + let context = IndexContext { groups }; + + debug_context(&report_path, &context); + + let text = self + .templates + .render("index", &context) + .expect("Failed to render index template"); + try_else_return!(fs::save_string(&text, &report_path,)); + } +} +impl Html { + fn comparison(&self, measurements: &MeasurementData<'_>) -> Option<Comparison> { + if let Some(ref comp) = measurements.comparison { + let different_mean = comp.p_value < comp.significance_threshold; + let mean_est = &comp.relative_estimates.mean; + let explanation_str: String; + + if !different_mean { + explanation_str = "No change in performance detected.".to_owned(); + } else { + let comparison = compare_to_threshold(&mean_est, comp.noise_threshold); + match comparison { + ComparisonResult::Improved => { + explanation_str = "Performance has improved.".to_owned(); + } + ComparisonResult::Regressed => { + explanation_str = "Performance has regressed.".to_owned(); + } + ComparisonResult::NonSignificant => { + explanation_str = "Change within noise threshold.".to_owned(); + } + } + } + + let comp = Comparison { + p_value: format!("{:.2}", comp.p_value), + inequality: (if different_mean { "<" } else { ">" }).to_owned(), + significance_level: format!("{:.2}", comp.significance_threshold), + explanation: explanation_str, + + change: ConfidenceInterval { + point: format::change(mean_est.point_estimate, true), + lower: format::change(mean_est.confidence_interval.lower_bound, true), + upper: format::change(mean_est.confidence_interval.upper_bound, true), + }, + + thrpt_change: measurements.throughput.as_ref().map(|_| { + let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0; + ConfidenceInterval { + point: format::change(to_thrpt_estimate(mean_est.point_estimate), true), + lower: format::change( + to_thrpt_estimate(mean_est.confidence_interval.lower_bound), + true, + ), + upper: format::change( + to_thrpt_estimate(mean_est.confidence_interval.upper_bound), + true, + ), + } + }), + + additional_plots: vec![ + Plot::new("Change in mean", "change/mean.svg"), + Plot::new("Change in median", "change/median.svg"), + Plot::new("T-Test", "change/t-test.svg"), + ], + }; + Some(comp) + } else { + None + } + } + + fn generate_plots( + &self, + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + ) { + let plot_ctx = PlotContext { + id, + context, + size: None, + is_thumbnail: false, + }; + + let plot_data = PlotData { + measurements, + formatter, + comparison: None, + }; + + let plot_ctx_small = plot_ctx.thumbnail(true).size(THUMBNAIL_SIZE); + + self.plotter.borrow_mut().pdf(plot_ctx, plot_data); + self.plotter.borrow_mut().pdf(plot_ctx_small, plot_data); + if measurements.absolute_estimates.slope.is_some() { + self.plotter.borrow_mut().regression(plot_ctx, plot_data); + self.plotter + .borrow_mut() + .regression(plot_ctx_small, plot_data); + } else { + self.plotter + .borrow_mut() + .iteration_times(plot_ctx, plot_data); + self.plotter + .borrow_mut() + .iteration_times(plot_ctx_small, plot_data); + } + + self.plotter + .borrow_mut() + .abs_distributions(plot_ctx, plot_data); + + if let Some(ref comp) = measurements.comparison { + try_else_return!({ + let mut change_dir = context.output_directory.clone(); + change_dir.push(id.as_directory_name()); + change_dir.push("report"); + change_dir.push("change"); + fs::mkdirp(&change_dir) + }); + + try_else_return!({ + let mut both_dir = context.output_directory.clone(); + both_dir.push(id.as_directory_name()); + both_dir.push("report"); + both_dir.push("both"); + fs::mkdirp(&both_dir) + }); + + let comp_data = plot_data.comparison(&comp); + + self.plotter.borrow_mut().pdf(plot_ctx, comp_data); + self.plotter.borrow_mut().pdf(plot_ctx_small, comp_data); + if measurements.absolute_estimates.slope.is_some() + && comp.base_estimates.slope.is_some() + { + self.plotter.borrow_mut().regression(plot_ctx, comp_data); + self.plotter + .borrow_mut() + .regression(plot_ctx_small, comp_data); + } else { + self.plotter + .borrow_mut() + .iteration_times(plot_ctx, comp_data); + self.plotter + .borrow_mut() + .iteration_times(plot_ctx_small, comp_data); + } + self.plotter.borrow_mut().t_test(plot_ctx, comp_data); + self.plotter + .borrow_mut() + .rel_distributions(plot_ctx, comp_data); + } + + self.plotter.borrow_mut().wait(); + } + + fn load_summary_data<'a>( + &self, + output_directory: &Path, + all_ids: &[&'a BenchmarkId], + ) -> Vec<(&'a BenchmarkId, Vec<f64>)> { + all_ids + .iter() + .filter_map(|id| { + let entry = output_directory.join(id.as_directory_name()).join("new"); + + let SavedSample { iters, times, .. } = + try_else_return!(fs::load(&entry.join("sample.json")), || None); + let avg_times = iters + .into_iter() + .zip(times.into_iter()) + .map(|(iters, time)| time / iters) + .collect::<Vec<_>>(); + + Some((*id, avg_times)) + }) + .collect::<Vec<_>>() + } + + fn generate_summary( + &self, + id: &BenchmarkId, + data: &[&(&BenchmarkId, Vec<f64>)], + report_context: &ReportContext, + formatter: &dyn ValueFormatter, + full_summary: bool, + ) { + let plot_ctx = PlotContext { + id, + context: report_context, + size: None, + is_thumbnail: false, + }; + + try_else_return!( + { + let mut report_dir = report_context.output_directory.clone(); + report_dir.push(id.as_directory_name()); + report_dir.push("report"); + fs::mkdirp(&report_dir) + }, + || {} + ); + + self.plotter.borrow_mut().violin(plot_ctx, formatter, data); + + let value_types: Vec<_> = data.iter().map(|&&(ref id, _)| id.value_type()).collect(); + let mut line_path = None; + + if value_types.iter().all(|x| x == &value_types[0]) { + if let Some(value_type) = value_types[0] { + let values: Vec<_> = data.iter().map(|&&(ref id, _)| id.as_number()).collect(); + if values.iter().any(|x| x != &values[0]) { + self.plotter + .borrow_mut() + .line_comparison(plot_ctx, formatter, data, value_type); + line_path = Some(plot_ctx.line_comparison_path()); + } + } + } + + let path_prefix = if full_summary { "../.." } else { "../../.." }; + let benchmarks = data + .iter() + .map(|&&(ref id, _)| { + IndividualBenchmark::from_id(&report_context.output_directory, path_prefix, id) + }) + .collect(); + + let context = SummaryContext { + group_id: id.as_title().to_owned(), + + thumbnail_width: THUMBNAIL_SIZE.unwrap().0, + thumbnail_height: THUMBNAIL_SIZE.unwrap().1, + + violin_plot: Some(plot_ctx.violin_path().to_string_lossy().into_owned()), + line_chart: line_path.map(|p| p.to_string_lossy().into_owned()), + + benchmarks, + }; + + let mut report_path = report_context.output_directory.clone(); + report_path.push(id.as_directory_name()); + report_path.push("report"); + report_path.push("index.html"); + debug_context(&report_path, &context); + + let text = self + .templates + .render("summary_report", &context) + .expect("Failed to render summary report template"); + try_else_return!(fs::save_string(&text, &report_path,), || {}); + } +} + +enum ComparisonResult { + Improved, + Regressed, + NonSignificant, +} + +fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult { + let ci = &estimate.confidence_interval; + let lb = ci.lower_bound; + let ub = ci.upper_bound; + + if lb < -noise && ub < -noise { + ComparisonResult::Improved + } else if lb > noise && ub > noise { + ComparisonResult::Regressed + } else { + ComparisonResult::NonSignificant + } +} diff --git a/src/html/report_link.html.tt b/src/html/report_link.html.tt new file mode 100755 index 0000000..6013114 --- /dev/null +++ b/src/html/report_link.html.tt @@ -0,0 +1,5 @@ +{{ if path -}}
+<a href="../{path}">{name}</a>
+{{- else -}}
+{name}
+{{- endif}}
\ No newline at end of file diff --git a/src/html/summary_report.html.tt b/src/html/summary_report.html.tt new file mode 100755 index 0000000..4f36f62 --- /dev/null +++ b/src/html/summary_report.html.tt @@ -0,0 +1,109 @@ +<!DOCTYPE html> +<html> + +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <title>{group_id} Summary - Criterion.rs</title> + <style type="text/css"> + body \{ + font: 14px Helvetica Neue; + text-rendering: optimizelegibility; + } + + .body \{ + width: 960px; + margin: auto; + } + + a:link \{ + color: #1F78B4; + text-decoration: none; + } + + h2 \{ + font-size: 36px; + font-weight: 300; + } + + h3 \{ + font-size: 24px; + font-weight: 300; + } + + #footer \{ + height: 40px; + background: #888; + color: white; + font-size: larger; + font-weight: 300; + } + + #footer a \{ + color: white; + text-decoration: underline; + } + + #footer p \{ + text-align: center + } + </style> +</head> + +<body> + <div class="body"> + <h2>{group_id}</h2> + {{- if violin_plot }} + <h3>Violin Plot</h3> + <a href="violin.svg"> + <img src="violin.svg" alt="Violin Plot" /> + </a> + <p>This chart shows the relationship between function/parameter and iteration time. The thickness of the shaded + region indicates the probability that a measurement of the given function/parameter would take a particular + length of time.</p> + {{- endif }} + {{- if line_chart }} + <h3>Line Chart</h3> + <img src="lines.svg" alt="Line Chart" /> + <p>This chart shows the mean measured time for each function as the input (or the size of the input) increases.</p> + {{- endif }} + {{- for bench in benchmarks }} + <section class="plots"> + <a href="{bench.path}/report/index.html"> + <h4>{bench.name}</h4> + </a> + <table width="100%"> + <tbody> + <tr> + <td> + <a href="{bench.path}/report/pdf.svg"> + <img src="{bench.path}/report/pdf_small.svg" alt="PDF of Slope" width="{thumbnail_width}" + height="{thumbnail_height}" /> + </a> + </td> + <td> + {{- if bench.regression_exists }} + <a href="{bench.path}/report/regression.svg"> + <img src="{bench.path}/report/regression_small.svg" alt="Regression" width="{thumbnail_width}" + height="{thumbnail_height}" /> + </a> + {{- else }} + <a href="{bench.path}/report/iteration_times.svg"> + <img src="{bench.path}/report/iteration_times_small.svg" alt="Iteration Times" width="{thumbnail_width}" + height="{thumbnail_height}" /> + </a> + {{- endif }} + </td> + </tr> + </tbody> + </table> + </section> + {{- endfor }} + </div> + <div id="footer"> + <p>This report was generated by + <a href="https://github.com/bheisler/criterion.rs">Criterion.rs</a>, a statistics-driven benchmarking + library in Rust.</p> + </div> +</body> + +</html>
\ No newline at end of file diff --git a/src/kde.rs b/src/kde.rs new file mode 100755 index 0000000..8812142 --- /dev/null +++ b/src/kde.rs @@ -0,0 +1,41 @@ +use crate::stats::univariate::kde::kernel::Gaussian; +use crate::stats::univariate::kde::{Bandwidth, Kde}; +use crate::stats::univariate::Sample; + +pub fn sweep( + sample: &Sample<f64>, + npoints: usize, + range: Option<(f64, f64)>, +) -> (Box<[f64]>, Box<[f64]>) { + let (xs, ys, _) = sweep_and_estimate(sample, npoints, range, sample[0]); + (xs, ys) +} + +pub fn sweep_and_estimate( + sample: &Sample<f64>, + npoints: usize, + range: Option<(f64, f64)>, + point_to_estimate: f64, +) -> (Box<[f64]>, Box<[f64]>, f64) { + let x_min = sample.min(); + let x_max = sample.max(); + + let kde = Kde::new(sample, Gaussian, Bandwidth::Silverman); + let h = kde.bandwidth(); + + let (start, end) = match range { + Some((start, end)) => (start, end), + None => (x_min - 3. * h, x_max + 3. * h), + }; + + let mut xs: Vec<f64> = Vec::with_capacity(npoints); + let step_size = (end - start) / (npoints - 1) as f64; + for n in 0..npoints { + xs.push(start + (step_size * n as f64)); + } + + let ys = kde.map(&xs); + let point_estimate = kde.estimate(point_to_estimate); + + (xs.into_boxed_slice(), ys, point_estimate) +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100755 index 0000000..0fd273c --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,1899 @@ +//! A statistics-driven micro-benchmarking library written in Rust. +//! +//! This crate is a microbenchmarking library which aims to provide strong +//! statistical confidence in detecting and estimating the size of performance +//! improvements and regressions, while also being easy to use. +//! +//! See +//! [the user guide](https://bheisler.github.io/criterion.rs/book/index.html) +//! for examples as well as details on the measurement and analysis process, +//! and the output. +//! +//! ## Features: +//! * Collects detailed statistics, providing strong confidence that changes +//! to performance are real, not measurement noise +//! * Produces detailed charts, providing thorough understanding of your code's +//! performance behavior. + +#![warn(missing_docs)] +#![warn(bare_trait_objects)] +#![cfg_attr(feature = "real_blackbox", feature(test))] +#![cfg_attr( + feature = "cargo-clippy", + allow( + clippy::just_underscores_and_digits, // Used in the stats code + clippy::transmute_ptr_to_ptr, // Used in the stats code + clippy::option_as_ref_deref, // Remove when MSRV bumped above 1.40 + ) +)] + +#[cfg(test)] +extern crate approx; + +#[cfg(test)] +extern crate quickcheck; + +use clap::value_t; +use regex::Regex; + +#[macro_use] +extern crate lazy_static; + +#[cfg(feature = "real_blackbox")] +extern crate test; + +#[macro_use] +extern crate serde_derive; + +// Needs to be declared before other modules +// in order to be usable there. +#[macro_use] +mod macros_private; +#[macro_use] +mod analysis; +mod benchmark; +#[macro_use] +mod benchmark_group; +mod connection; +mod csv_report; +mod error; +mod estimate; +mod format; +mod fs; +mod html; +mod kde; +mod macros; +pub mod measurement; +mod plot; +pub mod profiler; +mod report; +mod routine; +mod stats; + +use std::cell::RefCell; +use std::collections::HashSet; +use std::default::Default; +use std::fmt; +use std::iter::IntoIterator; +use std::marker::PhantomData; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; +use std::time::Instant; + +use criterion_plot::{Version, VersionError}; + +use crate::benchmark::BenchmarkConfig; +use crate::benchmark::NamedRoutine; +use crate::connection::Connection; +use crate::connection::OutgoingMessage; +use crate::csv_report::FileCsvReport; +use crate::html::Html; +use crate::measurement::{Measurement, WallTime}; +use crate::plot::{Gnuplot, Plotter, PlottersBackend}; +use crate::profiler::{ExternalProfiler, Profiler}; +use crate::report::{BencherReport, CliReport, Report, ReportContext, Reports}; +use crate::routine::Function; + +pub use crate::benchmark::{Benchmark, BenchmarkDefinition, ParameterizedBenchmark}; +pub use crate::benchmark_group::{BenchmarkGroup, BenchmarkId}; + +lazy_static! { + static ref DEBUG_ENABLED: bool = std::env::var_os("CRITERION_DEBUG").is_some(); + static ref GNUPLOT_VERSION: Result<Version, VersionError> = criterion_plot::version(); + static ref DEFAULT_PLOTTING_BACKEND: PlottingBackend = { + match &*GNUPLOT_VERSION { + Ok(_) => PlottingBackend::Gnuplot, + Err(e) => { + match e { + VersionError::Exec(_) => println!("Gnuplot not found, using plotters backend"), + e => println!( + "Gnuplot not found or not usable, using plotters backend\n{}", + e + ), + }; + PlottingBackend::Plotters + } + } + }; + static ref CARGO_CRITERION_CONNECTION: Option<Mutex<Connection>> = { + match std::env::var("CARGO_CRITERION_PORT") { + Ok(port_str) => { + let port: u16 = port_str.parse().ok()?; + let stream = TcpStream::connect(("localhost", port)).ok()?; + Some(Mutex::new(Connection::new(stream).ok()?)) + } + Err(_) => None, + } + }; +} + +fn debug_enabled() -> bool { + *DEBUG_ENABLED +} + +/// A function that is opaque to the optimizer, used to prevent the compiler from +/// optimizing away computations in a benchmark. +/// +/// This variant is backed by the (unstable) test::black_box function. +#[cfg(feature = "real_blackbox")] +pub fn black_box<T>(dummy: T) -> T { + test::black_box(dummy) +} + +/// A function that is opaque to the optimizer, used to prevent the compiler from +/// optimizing away computations in a benchmark. +/// +/// This variant is stable-compatible, but it may cause some performance overhead +/// or fail to prevent code from being eliminated. +#[cfg(not(feature = "real_blackbox"))] +pub fn black_box<T>(dummy: T) -> T { + unsafe { + let ret = std::ptr::read_volatile(&dummy); + std::mem::forget(dummy); + ret + } +} + +/// Representing a function to benchmark together with a name of that function. +/// Used together with `bench_functions` to represent one out of multiple functions +/// under benchmark. +#[doc(hidden)] +pub struct Fun<I: fmt::Debug, M: Measurement + 'static = WallTime> { + f: NamedRoutine<I, M>, + _phantom: PhantomData<M>, +} + +impl<I, M: Measurement> Fun<I, M> +where + I: fmt::Debug + 'static, +{ + /// Create a new `Fun` given a name and a closure + pub fn new<F>(name: &str, f: F) -> Fun<I, M> + where + F: FnMut(&mut Bencher<'_, M>, &I) + 'static, + { + let routine = NamedRoutine { + id: name.to_owned(), + f: Box::new(RefCell::new(Function::new(f))), + }; + + Fun { + f: routine, + _phantom: PhantomData, + } + } +} + +/// Argument to [`Bencher::iter_batched`](struct.Bencher.html#method.iter_batched) and +/// [`Bencher::iter_batched_ref`](struct.Bencher.html#method.iter_batched_ref) which controls the +/// batch size. +/// +/// Generally speaking, almost all benchmarks should use `SmallInput`. If the input or the result +/// of the benchmark routine is large enough that `SmallInput` causes out-of-memory errors, +/// `LargeInput` can be used to reduce memory usage at the cost of increasing the measurement +/// overhead. If the input or the result is extremely large (or if it holds some +/// limited external resource like a file handle), `PerIteration` will set the number of iterations +/// per batch to exactly one. `PerIteration` can increase the measurement overhead substantially +/// and should be avoided wherever possible. +/// +/// Each value lists an estimate of the measurement overhead. This is intended as a rough guide +/// to assist in choosing an option, it should not be relied upon. In particular, it is not valid +/// to subtract the listed overhead from the measurement and assume that the result represents the +/// true runtime of a function. The actual measurement overhead for your specific benchmark depends +/// on the details of the function you're benchmarking and the hardware and operating +/// system running the benchmark. +/// +/// With that said, if the runtime of your function is small relative to the measurement overhead +/// it will be difficult to take accurate measurements. In this situation, the best option is to use +/// [`Bencher::iter`](struct.Bencher.html#method.iter) which has next-to-zero measurement overhead. +#[derive(Debug, Eq, PartialEq, Copy, Hash, Clone)] +pub enum BatchSize { + /// `SmallInput` indicates that the input to the benchmark routine (the value returned from + /// the setup routine) is small enough that millions of values can be safely held in memory. + /// Always prefer `SmallInput` unless the benchmark is using too much memory. + /// + /// In testing, the maximum measurement overhead from benchmarking with `SmallInput` is on the + /// order of 500 picoseconds. This is presented as a rough guide; your results may vary. + SmallInput, + + /// `LargeInput` indicates that the input to the benchmark routine or the value returned from + /// that routine is large. This will reduce the memory usage but increase the measurement + /// overhead. + /// + /// In testing, the maximum measurement overhead from benchmarking with `LargeInput` is on the + /// order of 750 picoseconds. This is presented as a rough guide; your results may vary. + LargeInput, + + /// `PerIteration` indicates that the input to the benchmark routine or the value returned from + /// that routine is extremely large or holds some limited resource, such that holding many values + /// in memory at once is infeasible. This provides the worst measurement overhead, but the + /// lowest memory usage. + /// + /// In testing, the maximum measurement overhead from benchmarking with `PerIteration` is on the + /// order of 350 nanoseconds or 350,000 picoseconds. This is presented as a rough guide; your + /// results may vary. + PerIteration, + + /// `NumBatches` will attempt to divide the iterations up into a given number of batches. + /// A larger number of batches (and thus smaller batches) will reduce memory usage but increase + /// measurement overhead. This allows the user to choose their own tradeoff between memory usage + /// and measurement overhead, but care must be taken in tuning the number of batches. Most + /// benchmarks should use `SmallInput` or `LargeInput` instead. + NumBatches(u64), + + /// `NumIterations` fixes the batch size to a constant number, specified by the user. This + /// allows the user to choose their own tradeoff between overhead and memory usage, but care must + /// be taken in tuning the batch size. In general, the measurement overhead of NumIterations + /// will be larger than that of `NumBatches`. Most benchmarks should use `SmallInput` or + /// `LargeInput` instead. + NumIterations(u64), + + #[doc(hidden)] + __NonExhaustive, +} +impl BatchSize { + /// Convert to a number of iterations per batch. + /// + /// We try to do a constant number of batches regardless of the number of iterations in this + /// sample. If the measurement overhead is roughly constant regardless of the number of + /// iterations the analysis of the results later will have an easier time separating the + /// measurement overhead from the benchmark time. + fn iters_per_batch(self, iters: u64) -> u64 { + match self { + BatchSize::SmallInput => (iters + 10 - 1) / 10, + BatchSize::LargeInput => (iters + 1000 - 1) / 1000, + BatchSize::PerIteration => 1, + BatchSize::NumBatches(batches) => (iters + batches - 1) / batches, + BatchSize::NumIterations(size) => size, + BatchSize::__NonExhaustive => panic!("__NonExhaustive is not a valid BatchSize."), + } + } +} + +/// Timer struct used to iterate a benchmarked function and measure the runtime. +/// +/// This struct provides different timing loops as methods. Each timing loop provides a different +/// way to time a routine and each has advantages and disadvantages. +/// +/// * If you want to do the iteration and measurement yourself (eg. passing the iteration count +/// to a separate process), use `iter_custom`. +/// * If your routine requires no per-iteration setup and returns a value with an expensive `drop` +/// method, use `iter_with_large_drop`. +/// * If your routine requires some per-iteration setup that shouldn't be timed, use `iter_batched` +/// or `iter_batched_ref`. See [`BatchSize`](enum.BatchSize.html) for a discussion of batch sizes. +/// If the setup value implements `Drop` and you don't want to include the `drop` time in the +/// measurement, use `iter_batched_ref`, otherwise use `iter_batched`. These methods are also +/// suitable for benchmarking routines which return a value with an expensive `drop` method, +/// but are more complex than `iter_with_large_drop`. +/// * Otherwise, use `iter`. +pub struct Bencher<'a, M: Measurement = WallTime> { + iterated: bool, // have we iterated this benchmark? + iters: u64, // Number of times to iterate this benchmark + value: M::Value, // The measured value + measurement: &'a M, // Reference to the measurement object + elapsed_time: Duration, // How much time did it take to perform the iteration? Used for the warmup period. +} +impl<'a, M: Measurement> Bencher<'a, M> { + /// Times a `routine` by executing it many times and timing the total elapsed time. + /// + /// Prefer this timing loop when `routine` returns a value that doesn't have a destructor. + /// + /// # Timing model + /// + /// Note that the `Bencher` also times the time required to destroy the output of `routine()`. + /// Therefore prefer this timing loop when the runtime of `mem::drop(O)` is negligible compared + /// to the runtime of the `routine`. + /// + /// ```text + /// elapsed = Instant::now + iters * (routine + mem::drop(O) + Range::next) + /// ``` + /// + /// # Example + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// + /// use criterion::*; + /// + /// // The function to benchmark + /// fn foo() { + /// // ... + /// } + /// + /// fn bench(c: &mut Criterion) { + /// c.bench_function("iter", move |b| { + /// b.iter(|| foo()) + /// }); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + /// + #[inline(never)] + pub fn iter<O, R>(&mut self, mut routine: R) + where + R: FnMut() -> O, + { + self.iterated = true; + let time_start = Instant::now(); + let start = self.measurement.start(); + for _ in 0..self.iters { + black_box(routine()); + } + self.value = self.measurement.end(start); + self.elapsed_time = time_start.elapsed(); + } + + /// Times a `routine` by executing it many times and relying on `routine` to measure its own execution time. + /// + /// Prefer this timing loop in cases where `routine` has to do its own measurements to + /// get accurate timing information (for example in multi-threaded scenarios where you spawn + /// and coordinate with multiple threads). + /// + /// # Timing model + /// Custom, the timing model is whatever is returned as the Duration from `routine`. + /// + /// # Example + /// ```rust + /// #[macro_use] extern crate criterion; + /// use criterion::*; + /// use criterion::black_box; + /// use std::time::Instant; + /// + /// fn foo() { + /// // ... + /// } + /// + /// fn bench(c: &mut Criterion) { + /// c.bench_function("iter", move |b| { + /// b.iter_custom(|iters| { + /// let start = Instant::now(); + /// for _i in 0..iters { + /// black_box(foo()); + /// } + /// start.elapsed() + /// }) + /// }); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + /// + #[inline(never)] + pub fn iter_custom<R>(&mut self, mut routine: R) + where + R: FnMut(u64) -> M::Value, + { + self.iterated = true; + let time_start = Instant::now(); + self.value = routine(self.iters); + self.elapsed_time = time_start.elapsed(); + } + + #[doc(hidden)] + pub fn iter_with_setup<I, O, S, R>(&mut self, setup: S, routine: R) + where + S: FnMut() -> I, + R: FnMut(I) -> O, + { + self.iter_batched(setup, routine, BatchSize::PerIteration); + } + + /// Times a `routine` by collecting its output on each iteration. This avoids timing the + /// destructor of the value returned by `routine`. + /// + /// WARNING: This requires `O(iters * mem::size_of::<O>())` of memory, and `iters` is not under the + /// control of the caller. If this causes out-of-memory errors, use `iter_batched` instead. + /// + /// # Timing model + /// + /// ``` text + /// elapsed = Instant::now + iters * (routine) + Iterator::collect::<Vec<_>> + /// ``` + /// + /// # Example + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// + /// use criterion::*; + /// + /// fn create_vector() -> Vec<u64> { + /// # vec![] + /// // ... + /// } + /// + /// fn bench(c: &mut Criterion) { + /// c.bench_function("with_drop", move |b| { + /// // This will avoid timing the Vec::drop. + /// b.iter_with_large_drop(|| create_vector()) + /// }); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + /// + pub fn iter_with_large_drop<O, R>(&mut self, mut routine: R) + where + R: FnMut() -> O, + { + self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput); + } + + #[doc(hidden)] + pub fn iter_with_large_setup<I, O, S, R>(&mut self, setup: S, routine: R) + where + S: FnMut() -> I, + R: FnMut(I) -> O, + { + self.iter_batched(setup, routine, BatchSize::NumBatches(1)); + } + + /// Times a `routine` that requires some input by generating a batch of input, then timing the + /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for + /// details on choosing the batch size. Use this when the routine must consume its input. + /// + /// For example, use this loop to benchmark sorting algorithms, because they require unsorted + /// data on each iteration. + /// + /// # Timing model + /// + /// ```text + /// elapsed = (Instant::now * num_batches) + (iters * (routine + O::drop)) + Vec::extend + /// ``` + /// + /// # Example + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// + /// use criterion::*; + /// + /// fn create_scrambled_data() -> Vec<u64> { + /// # vec![] + /// // ... + /// } + /// + /// // The sorting algorithm to test + /// fn sort(data: &mut [u64]) { + /// // ... + /// } + /// + /// fn bench(c: &mut Criterion) { + /// let data = create_scrambled_data(); + /// + /// c.bench_function("with_setup", move |b| { + /// // This will avoid timing the to_vec call. + /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput) + /// }); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + /// + #[inline(never)] + pub fn iter_batched<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize) + where + S: FnMut() -> I, + R: FnMut(I) -> O, + { + self.iterated = true; + let batch_size = size.iters_per_batch(self.iters); + assert!(batch_size != 0, "Batch size must not be zero."); + let time_start = Instant::now(); + self.value = self.measurement.zero(); + + if batch_size == 1 { + for _ in 0..self.iters { + let input = black_box(setup()); + + let start = self.measurement.start(); + let output = routine(input); + let end = self.measurement.end(start); + self.value = self.measurement.add(&self.value, &end); + + drop(black_box(output)); + } + } else { + let mut iteration_counter = 0; + + while iteration_counter < self.iters { + let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter); + + let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>()); + let mut outputs = Vec::with_capacity(batch_size as usize); + + let start = self.measurement.start(); + outputs.extend(inputs.into_iter().map(&mut routine)); + let end = self.measurement.end(start); + self.value = self.measurement.add(&self.value, &end); + + black_box(outputs); + + iteration_counter += batch_size; + } + } + + self.elapsed_time = time_start.elapsed(); + } + + /// Times a `routine` that requires some input by generating a batch of input, then timing the + /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for + /// details on choosing the batch size. Use this when the routine should accept the input by + /// mutable reference. + /// + /// For example, use this loop to benchmark sorting algorithms, because they require unsorted + /// data on each iteration. + /// + /// # Timing model + /// + /// ```text + /// elapsed = (Instant::now * num_batches) + (iters * routine) + Vec::extend + /// ``` + /// + /// # Example + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// + /// use criterion::*; + /// + /// fn create_scrambled_data() -> Vec<u64> { + /// # vec![] + /// // ... + /// } + /// + /// // The sorting algorithm to test + /// fn sort(data: &mut [u64]) { + /// // ... + /// } + /// + /// fn bench(c: &mut Criterion) { + /// let data = create_scrambled_data(); + /// + /// c.bench_function("with_setup", move |b| { + /// // This will avoid timing the to_vec call. + /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput) + /// }); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + /// + #[inline(never)] + pub fn iter_batched_ref<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize) + where + S: FnMut() -> I, + R: FnMut(&mut I) -> O, + { + self.iterated = true; + let batch_size = size.iters_per_batch(self.iters); + assert!(batch_size != 0, "Batch size must not be zero."); + let time_start = Instant::now(); + self.value = self.measurement.zero(); + + if batch_size == 1 { + for _ in 0..self.iters { + let mut input = black_box(setup()); + + let start = self.measurement.start(); + let output = routine(&mut input); + let end = self.measurement.end(start); + self.value = self.measurement.add(&self.value, &end); + + drop(black_box(output)); + drop(black_box(input)); + } + } else { + let mut iteration_counter = 0; + + while iteration_counter < self.iters { + let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter); + + let mut inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>()); + let mut outputs = Vec::with_capacity(batch_size as usize); + + let start = self.measurement.start(); + outputs.extend(inputs.iter_mut().map(&mut routine)); + let end = self.measurement.end(start); + self.value = self.measurement.add(&self.value, &end); + + black_box(outputs); + + iteration_counter += batch_size; + } + } + self.elapsed_time = time_start.elapsed(); + } + + // Benchmarks must actually call one of the iter methods. This causes benchmarks to fail loudly + // if they don't. + fn assert_iterated(&mut self) { + if !self.iterated { + panic!("Benchmark function must call Bencher::iter or related method."); + } + self.iterated = false; + } +} + +/// Baseline describes how the baseline_directory is handled. +#[derive(Debug, Clone, Copy)] +pub enum Baseline { + /// Compare ensures a previous saved version of the baseline + /// exists and runs comparison against that. + Compare, + /// Save writes the benchmark results to the baseline directory, + /// overwriting any results that were previously there. + Save, +} + +/// Enum used to select the plotting backend. +#[derive(Debug, Clone, Copy)] +pub enum PlottingBackend { + /// Plotting backend which uses the external `gnuplot` command to render plots. This is the + /// default if the `gnuplot` command is installed. + Gnuplot, + /// Plotting backend which uses the rust 'Plotters' library. This is the default if `gnuplot` + /// is not installed. + Plotters, +} + +#[derive(Debug, Clone)] +/// Enum representing the execution mode. +pub(crate) enum Mode { + /// Run benchmarks normally + Benchmark, + /// List all benchmarks but do not run them. + List, + /// Run bennchmarks once to verify that they work, but otherwise do not measure them. + Test, + /// Iterate benchmarks for a given length of time but do not analyze or report on them. + Profile(Duration), +} +impl Mode { + pub fn is_benchmark(&self) -> bool { + match self { + Mode::Benchmark => true, + _ => false, + } + } +} + +/// The benchmark manager +/// +/// `Criterion` lets you configure and execute benchmarks +/// +/// Each benchmark consists of four phases: +/// +/// - **Warm-up**: The routine is repeatedly executed, to let the CPU/OS/JIT/interpreter adapt to +/// the new load +/// - **Measurement**: The routine is repeatedly executed, and timing information is collected into +/// a sample +/// - **Analysis**: The sample is analyzed and distiled into meaningful statistics that get +/// reported to stdout, stored in files, and plotted +/// - **Comparison**: The current sample is compared with the sample obtained in the previous +/// benchmark. +pub struct Criterion<M: Measurement = WallTime> { + config: BenchmarkConfig, + plotting_backend: PlottingBackend, + plotting_enabled: bool, + filter: Option<Regex>, + report: Box<dyn Report>, + output_directory: PathBuf, + baseline_directory: String, + baseline: Baseline, + load_baseline: Option<String>, + all_directories: HashSet<String>, + all_titles: HashSet<String>, + measurement: M, + profiler: Box<RefCell<dyn Profiler>>, + connection: Option<MutexGuard<'static, Connection>>, + mode: Mode, +} + +impl Default for Criterion { + /// Creates a benchmark manager with the following default settings: + /// + /// - Sample size: 100 measurements + /// - Warm-up time: 3 s + /// - Measurement time: 5 s + /// - Bootstrap size: 100 000 resamples + /// - Noise threshold: 0.01 (1%) + /// - Confidence level: 0.95 + /// - Significance level: 0.05 + /// - Plotting: enabled, using gnuplot if available or plotters if gnuplot is not available + /// - No filter + fn default() -> Criterion { + let mut reports: Vec<Box<dyn Report>> = vec![]; + if CARGO_CRITERION_CONNECTION.is_none() { + reports.push(Box::new(CliReport::new(false, false, false))); + } + reports.push(Box::new(FileCsvReport)); + + // Set criterion home to (in descending order of preference): + // - $CRITERION_HOME (cargo-criterion sets this, but other users could as well) + // - $CARGO_TARGET_DIR/criterion + // - ./target/criterion + let output_directory = if let Some(value) = std::env::var_os("CRITERION_HOME") { + PathBuf::from(value) + } else if let Some(value) = std::env::var_os("CARGO_TARGET_DIR") { + PathBuf::from(value).join("criterion") + } else { + PathBuf::from("target/criterion") + }; + + Criterion { + config: BenchmarkConfig { + confidence_level: 0.95, + measurement_time: Duration::new(5, 0), + noise_threshold: 0.01, + nresamples: 100_000, + sample_size: 100, + significance_level: 0.05, + warm_up_time: Duration::new(3, 0), + sampling_mode: SamplingMode::Auto, + }, + plotting_backend: *DEFAULT_PLOTTING_BACKEND, + plotting_enabled: true, + filter: None, + report: Box::new(Reports::new(reports)), + baseline_directory: "base".to_owned(), + baseline: Baseline::Save, + load_baseline: None, + output_directory, + all_directories: HashSet::new(), + all_titles: HashSet::new(), + measurement: WallTime, + profiler: Box::new(RefCell::new(ExternalProfiler)), + connection: CARGO_CRITERION_CONNECTION + .as_ref() + .map(|mtx| mtx.lock().unwrap()), + mode: Mode::Benchmark, + } + } +} + +impl<M: Measurement> Criterion<M> { + /// Changes the measurement for the benchmarks run with this runner. See the + /// Measurement trait for more details + pub fn with_measurement<M2: Measurement>(self, m: M2) -> Criterion<M2> { + // Can't use struct update syntax here because they're technically different types. + Criterion { + config: self.config, + plotting_backend: self.plotting_backend, + plotting_enabled: self.plotting_enabled, + filter: self.filter, + report: self.report, + baseline_directory: self.baseline_directory, + baseline: self.baseline, + load_baseline: self.load_baseline, + output_directory: self.output_directory, + all_directories: self.all_directories, + all_titles: self.all_titles, + measurement: m, + profiler: self.profiler, + connection: self.connection, + mode: self.mode, + } + } + + /// Changes the internal profiler for benchmarks run with this runner. See + /// the Profiler trait for more details. + pub fn with_profiler<P: Profiler + 'static>(self, p: P) -> Criterion<M> { + Criterion { + profiler: Box::new(RefCell::new(p)), + ..self + } + } + + /// Set the plotting backend. By default, Criterion will use gnuplot if available, or plotters + /// if not. + /// + /// Panics if `backend` is `PlottingBackend::Gnuplot` and gnuplot is not available. + pub fn plotting_backend(self, backend: PlottingBackend) -> Criterion<M> { + if let PlottingBackend::Gnuplot = backend { + if GNUPLOT_VERSION.is_err() { + panic!("Gnuplot plotting backend was requested, but gnuplot is not available. To continue, either install Gnuplot or allow Criterion.rs to fall back to using plotters."); + } + } + + Criterion { + plotting_backend: backend, + ..self + } + } + + /// Changes the default size of the sample for benchmarks run with this runner. + /// + /// A bigger sample should yield more accurate results if paired with a sufficiently large + /// measurement time. + /// + /// Sample size must be at least 10. + /// + /// # Panics + /// + /// Panics if n < 10 + pub fn sample_size(mut self, n: usize) -> Criterion<M> { + assert!(n >= 10); + + self.config.sample_size = n; + self + } + + /// Changes the default warm up time for benchmarks run with this runner. + /// + /// # Panics + /// + /// Panics if the input duration is zero + pub fn warm_up_time(mut self, dur: Duration) -> Criterion<M> { + assert!(dur.to_nanos() > 0); + + self.config.warm_up_time = dur; + self + } + + /// Changes the default measurement time for benchmarks run with this runner. + /// + /// With a longer time, the measurement will become more resilient to transitory peak loads + /// caused by external programs + /// + /// **Note**: If the measurement time is too "low", Criterion will automatically increase it + /// + /// # Panics + /// + /// Panics if the input duration in zero + pub fn measurement_time(mut self, dur: Duration) -> Criterion<M> { + assert!(dur.to_nanos() > 0); + + self.config.measurement_time = dur; + self + } + + /// Changes the default number of resamples for benchmarks run with this runner. + /// + /// Number of resamples to use for the + /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling) + /// + /// A larger number of resamples reduces the random sampling errors, which are inherent to the + /// bootstrap method, but also increases the analysis time + /// + /// # Panics + /// + /// Panics if the number of resamples is set to zero + pub fn nresamples(mut self, n: usize) -> Criterion<M> { + assert!(n > 0); + if n <= 1000 { + println!("\nWarning: It is not recommended to reduce nresamples below 1000."); + } + + self.config.nresamples = n; + self + } + + /// Changes the default noise threshold for benchmarks run with this runner. The noise threshold + /// is used to filter out small changes in performance, even if they are statistically + /// significant. Sometimes benchmarking the same code twice will result in small but + /// statistically significant differences solely because of noise. This provides a way to filter + /// out some of these false positives at the cost of making it harder to detect small changes + /// to the true performance of the benchmark. + /// + /// The default is 0.01, meaning that changes smaller than 1% will be ignored. + /// + /// # Panics + /// + /// Panics if the threshold is set to a negative value + pub fn noise_threshold(mut self, threshold: f64) -> Criterion<M> { + assert!(threshold >= 0.0); + + self.config.noise_threshold = threshold; + self + } + + /// Changes the default confidence level for benchmarks run with this runner. The confidence + /// level is the desired probability that the true runtime lies within the estimated + /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is + /// 0.95, meaning that the confidence interval should capture the true value 95% of the time. + /// + /// # Panics + /// + /// Panics if the confidence level is set to a value outside the `(0, 1)` range + pub fn confidence_level(mut self, cl: f64) -> Criterion<M> { + assert!(cl > 0.0 && cl < 1.0); + if cl < 0.5 { + println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); + } + + self.config.confidence_level = cl; + self + } + + /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance) + /// for benchmarks run with this runner. This is used to perform a + /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if + /// the measurements from this run are different from the measured performance of the last run. + /// The significance level is the desired probability that two measurements of identical code + /// will be considered 'different' due to noise in the measurements. The default value is 0.05, + /// meaning that approximately 5% of identical benchmarks will register as different due to + /// noise. + /// + /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase + /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to + /// detect small but real changes in the performance. By setting the significance level + /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also + /// report more spurious differences. + /// + /// See also the noise threshold setting. + /// + /// # Panics + /// + /// Panics if the significance level is set to a value outside the `(0, 1)` range + pub fn significance_level(mut self, sl: f64) -> Criterion<M> { + assert!(sl > 0.0 && sl < 1.0); + + self.config.significance_level = sl; + self + } + + fn create_plotter(&self) -> Box<dyn Plotter> { + match self.plotting_backend { + PlottingBackend::Gnuplot => Box::new(Gnuplot::default()), + PlottingBackend::Plotters => Box::new(PlottersBackend::default()), + } + } + + /// Enables plotting + pub fn with_plots(mut self) -> Criterion<M> { + self.plotting_enabled = true; + let mut reports: Vec<Box<dyn Report>> = vec![]; + if self.connection.is_none() { + reports.push(Box::new(CliReport::new(false, false, false))); + } + reports.push(Box::new(FileCsvReport)); + reports.push(Box::new(Html::new(self.create_plotter()))); + self.report = Box::new(Reports::new(reports)); + + self + } + + /// Disables plotting + pub fn without_plots(mut self) -> Criterion<M> { + self.plotting_enabled = false; + let mut reports: Vec<Box<dyn Report>> = vec![]; + if self.connection.is_none() { + reports.push(Box::new(CliReport::new(false, false, false))); + } + reports.push(Box::new(FileCsvReport)); + self.report = Box::new(Reports::new(reports)); + self + } + + /// Return true if generation of the plots is possible. + pub fn can_plot(&self) -> bool { + // Trivially true now that we have plotters. + // TODO: Deprecate and remove this. + true + } + + /// Names an explicit baseline and enables overwriting the previous results. + pub fn save_baseline(mut self, baseline: String) -> Criterion<M> { + self.baseline_directory = baseline; + self.baseline = Baseline::Save; + self + } + + /// Names an explicit baseline and disables overwriting the previous results. + pub fn retain_baseline(mut self, baseline: String) -> Criterion<M> { + self.baseline_directory = baseline; + self.baseline = Baseline::Compare; + self + } + + /// Filters the benchmarks. Only benchmarks with names that contain the + /// given string will be executed. + pub fn with_filter<S: Into<String>>(mut self, filter: S) -> Criterion<M> { + let filter_text = filter.into(); + let filter = Regex::new(&filter_text).unwrap_or_else(|err| { + panic!( + "Unable to parse '{}' as a regular expression: {}", + filter_text, err + ) + }); + self.filter = Some(filter); + + self + } + + /// Set the output directory (currently for testing only) + #[doc(hidden)] + pub fn output_directory(mut self, path: &Path) -> Criterion<M> { + self.output_directory = path.to_owned(); + + self + } + + /// Set the profile time (currently for testing only) + #[doc(hidden)] + pub fn profile_time(mut self, profile_time: Option<Duration>) -> Criterion<M> { + match profile_time { + Some(time) => self.mode = Mode::Profile(time), + None => self.mode = Mode::Benchmark, + } + + self + } + + /// Generate the final summary at the end of a run. + #[doc(hidden)] + pub fn final_summary(&self) { + if !self.mode.is_benchmark() { + return; + } + + let report_context = ReportContext { + output_directory: self.output_directory.clone(), + plot_config: PlotConfiguration::default(), + }; + + self.report.final_summary(&report_context); + } + + /// Configure this criterion struct based on the command-line arguments to + /// this process. + #[cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))] + pub fn configure_from_args(mut self) -> Criterion<M> { + use clap::{App, Arg}; + let matches = App::new("Criterion Benchmark") + .arg(Arg::with_name("FILTER") + .help("Skip benchmarks whose names do not contain FILTER.") + .index(1)) + .arg(Arg::with_name("color") + .short("c") + .long("color") + .alias("colour") + .takes_value(true) + .possible_values(&["auto", "always", "never"]) + .default_value("auto") + .help("Configure coloring of output. always = always colorize output, never = never colorize output, auto = colorize output if output is a tty and compiled for unix.")) + .arg(Arg::with_name("verbose") + .short("v") + .long("verbose") + .help("Print additional statistical information.")) + .arg(Arg::with_name("noplot") + .short("n") + .long("noplot") + .help("Disable plot and HTML generation.")) + .arg(Arg::with_name("save-baseline") + .short("s") + .long("save-baseline") + .default_value("base") + .help("Save results under a named baseline.")) + .arg(Arg::with_name("baseline") + .short("b") + .long("baseline") + .takes_value(true) + .conflicts_with("save-baseline") + .help("Compare to a named baseline.")) + .arg(Arg::with_name("list") + .long("list") + .help("List all benchmarks") + .conflicts_with_all(&["test", "profile-time"])) + .arg(Arg::with_name("profile-time") + .long("profile-time") + .takes_value(true) + .help("Iterate each benchmark for approximately the given number of seconds, doing no analysis and without storing the results. Useful for running the benchmarks in a profiler.") + .conflicts_with_all(&["test", "list"])) + .arg(Arg::with_name("load-baseline") + .long("load-baseline") + .takes_value(true) + .conflicts_with("profile-time") + .requires("baseline") + .help("Load a previous baseline instead of sampling new data.")) + .arg(Arg::with_name("sample-size") + .long("sample-size") + .takes_value(true) + .help(&format!("Changes the default size of the sample for this run. [default: {}]", self.config.sample_size))) + .arg(Arg::with_name("warm-up-time") + .long("warm-up-time") + .takes_value(true) + .help(&format!("Changes the default warm up time for this run. [default: {}]", self.config.warm_up_time.as_secs()))) + .arg(Arg::with_name("measurement-time") + .long("measurement-time") + .takes_value(true) + .help(&format!("Changes the default measurement time for this run. [default: {}]", self.config.measurement_time.as_secs()))) + .arg(Arg::with_name("nresamples") + .long("nresamples") + .takes_value(true) + .help(&format!("Changes the default number of resamples for this run. [default: {}]", self.config.nresamples))) + .arg(Arg::with_name("noise-threshold") + .long("noise-threshold") + .takes_value(true) + .help(&format!("Changes the default noise threshold for this run. [default: {}]", self.config.noise_threshold))) + .arg(Arg::with_name("confidence-level") + .long("confidence-level") + .takes_value(true) + .help(&format!("Changes the default confidence level for this run. [default: {}]", self.config.confidence_level))) + .arg(Arg::with_name("significance-level") + .long("significance-level") + .takes_value(true) + .help(&format!("Changes the default significance level for this run. [default: {}]", self.config.significance_level))) + .arg(Arg::with_name("test") + .hidden(true) + .long("test") + .help("Run the benchmarks once, to verify that they execute successfully, but do not measure or report the results.") + .conflicts_with_all(&["list", "profile-time"])) + .arg(Arg::with_name("bench") + .hidden(true) + .long("bench")) + .arg(Arg::with_name("plotting-backend") + .long("plotting-backend") + .takes_value(true) + .possible_values(&["gnuplot", "plotters"]) + .help("Set the plotting backend. By default, Criterion.rs will use the gnuplot backend if gnuplot is available, or the plotters backend if it isn't.")) + .arg(Arg::with_name("output-format") + .long("output-format") + .takes_value(true) + .possible_values(&["criterion", "bencher"]) + .default_value("criterion") + .help("Change the CLI output format. By default, Criterion.rs will use its own format. If output format is set to 'bencher', Criterion.rs will print output in a format that resembles the 'bencher' crate.")) + .arg(Arg::with_name("nocapture") + .long("nocapture") + .hidden(true) + .help("Ignored, but added for compatibility with libtest.")) + .arg(Arg::with_name("version") + .hidden(true) + .short("V") + .long("version")) + .after_help(" +This executable is a Criterion.rs benchmark. +See https://github.com/bheisler/criterion.rs for more details. + +To enable debug output, define the environment variable CRITERION_DEBUG. +Criterion.rs will output more debug information and will save the gnuplot +scripts alongside the generated plots. + +To test that the benchmarks work, run `cargo test --benches` +") + .get_matches(); + + if self.connection.is_some() { + if let Some(color) = matches.value_of("color") { + if color != "auto" { + println!("Warning: --color will be ignored when running with cargo-criterion. Use `cargo criterion --color {} -- <args>` instead.", color); + } + } + if matches.is_present("verbose") { + println!("Warning: --verbose will be ignored when running with cargo-criterion. Use `cargo criterion --output-format verbose -- <args>` instead."); + } + if matches.is_present("noplot") { + println!("Warning: --noplot will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend disabled -- <args>` instead."); + } + if let Some(backend) = matches.value_of("plotting-backend") { + println!("Warning: --plotting-backend will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend {} -- <args>` instead.", backend); + } + if let Some(format) = matches.value_of("output-format") { + if format != "criterion" { + println!("Warning: --output-format will be ignored when running with cargo-criterion. Use `cargo criterion --output-format {} -- <args>` instead.", format); + } + } + + if matches.is_present("baseline") + || matches + .value_of("save-baseline") + .map(|base| base != "base") + .unwrap_or(false) + || matches.is_present("load-baseline") + { + println!("Error: baselines are not supported when running with cargo-criterion."); + std::process::exit(1); + } + } + + let bench = matches.is_present("bench"); + let test = matches.is_present("test"); + let test_mode = match (bench, test) { + (true, true) => true, // cargo bench -- --test should run tests + (true, false) => false, // cargo bench should run benchmarks + (false, _) => true, // cargo test --benches should run tests + }; + + self.mode = if test_mode { + Mode::Test + } else if matches.is_present("list") { + Mode::List + } else if matches.is_present("profile-time") { + let num_seconds = value_t!(matches.value_of("profile-time"), u64).unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + if num_seconds < 1 { + println!("Profile time must be at least one second."); + std::process::exit(1); + } + + Mode::Profile(Duration::from_secs(num_seconds)) + } else { + Mode::Benchmark + }; + + // This is kind of a hack, but disable the connection to the runner if we're not benchmarking. + if !self.mode.is_benchmark() { + self.connection = None; + } + + if let Some(filter) = matches.value_of("FILTER") { + self = self.with_filter(filter); + } + + match matches.value_of("plotting-backend") { + // Use plotting_backend() here to re-use the panic behavior if Gnuplot is not available. + Some("gnuplot") => self = self.plotting_backend(PlottingBackend::Gnuplot), + Some("plotters") => self = self.plotting_backend(PlottingBackend::Plotters), + Some(val) => panic!("Unexpected plotting backend '{}'", val), + None => {} + } + + if matches.is_present("noplot") { + self = self.without_plots(); + } else { + self = self.with_plots(); + } + + if let Some(dir) = matches.value_of("save-baseline") { + self.baseline = Baseline::Save; + self.baseline_directory = dir.to_owned() + } + if let Some(dir) = matches.value_of("baseline") { + self.baseline = Baseline::Compare; + self.baseline_directory = dir.to_owned(); + } + + if self.connection.is_some() { + // disable all reports when connected to cargo-criterion; it will do the reporting. + self.report = Box::new(Reports::new(vec![])); + } else { + let cli_report: Box<dyn Report> = match matches.value_of("output-format") { + Some("bencher") => Box::new(BencherReport), + _ => { + let verbose = matches.is_present("verbose"); + let stdout_isatty = atty::is(atty::Stream::Stdout); + let mut enable_text_overwrite = stdout_isatty && !verbose && !debug_enabled(); + let enable_text_coloring; + match matches.value_of("color") { + Some("always") => { + enable_text_coloring = true; + } + Some("never") => { + enable_text_coloring = false; + enable_text_overwrite = false; + } + _ => enable_text_coloring = stdout_isatty, + }; + Box::new(CliReport::new( + enable_text_overwrite, + enable_text_coloring, + verbose, + )) + } + }; + + let mut reports: Vec<Box<dyn Report>> = vec![]; + reports.push(cli_report); + reports.push(Box::new(FileCsvReport)); + if self.plotting_enabled { + reports.push(Box::new(Html::new(self.create_plotter()))); + } + self.report = Box::new(Reports::new(reports)); + } + + if let Some(dir) = matches.value_of("load-baseline") { + self.load_baseline = Some(dir.to_owned()); + } + + if matches.is_present("sample-size") { + let num_size = value_t!(matches.value_of("sample-size"), usize).unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + assert!(num_size >= 10); + self.config.sample_size = num_size; + } + if matches.is_present("warm-up-time") { + let num_seconds = value_t!(matches.value_of("warm-up-time"), u64).unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + let dur = std::time::Duration::new(num_seconds, 0); + assert!(dur.to_nanos() > 0); + + self.config.warm_up_time = dur; + } + if matches.is_present("measurement-time") { + let num_seconds = + value_t!(matches.value_of("measurement-time"), u64).unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + let dur = std::time::Duration::new(num_seconds, 0); + assert!(dur.to_nanos() > 0); + + self.config.measurement_time = dur; + } + if matches.is_present("nresamples") { + let num_resamples = + value_t!(matches.value_of("nresamples"), usize).unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + assert!(num_resamples > 0); + + self.config.nresamples = num_resamples; + } + if matches.is_present("noise-threshold") { + let num_noise_threshold = value_t!(matches.value_of("noise-threshold"), f64) + .unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + assert!(num_noise_threshold > 0.0); + + self.config.noise_threshold = num_noise_threshold; + } + if matches.is_present("confidence-level") { + let num_confidence_level = value_t!(matches.value_of("confidence-level"), f64) + .unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + assert!(num_confidence_level > 0.0 && num_confidence_level < 1.0); + + self.config.confidence_level = num_confidence_level; + } + if matches.is_present("significance-level") { + let num_significance_level = value_t!(matches.value_of("significance-level"), f64) + .unwrap_or_else(|e| { + println!("{}", e); + std::process::exit(1) + }); + + assert!(num_significance_level > 0.0 && num_significance_level < 1.0); + + self.config.significance_level = num_significance_level; + } + + self + } + + fn filter_matches(&self, id: &str) -> bool { + match self.filter { + Some(ref regex) => regex.is_match(id), + None => true, + } + } + + /// Return a benchmark group. All benchmarks performed using a benchmark group will be + /// grouped together in the final report. + /// + /// # Examples: + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// use self::criterion::*; + /// + /// fn bench_simple(c: &mut Criterion) { + /// let mut group = c.benchmark_group("My Group"); + /// + /// // Now we can perform benchmarks with this group + /// group.bench_function("Bench 1", |b| b.iter(|| 1 )); + /// group.bench_function("Bench 2", |b| b.iter(|| 2 )); + /// + /// group.finish(); + /// } + /// criterion_group!(benches, bench_simple); + /// criterion_main!(benches); + /// ``` + /// # Panics: + /// Panics if the group name is empty + pub fn benchmark_group<S: Into<String>>(&mut self, group_name: S) -> BenchmarkGroup<'_, M> { + let group_name = group_name.into(); + if group_name.is_empty() { + panic!("Group name must not be empty."); + } + + if let Some(conn) = &self.connection { + conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: &group_name }) + .unwrap(); + } + + BenchmarkGroup::new(self, group_name) + } +} +impl<M> Criterion<M> +where + M: Measurement + 'static, +{ + /// Benchmarks a function. For comparing multiple functions, see `benchmark_group`. + /// + /// # Example + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// use self::criterion::*; + /// + /// fn bench(c: &mut Criterion) { + /// // Setup (construct data, allocate memory, etc) + /// c.bench_function( + /// "function_name", + /// |b| b.iter(|| { + /// // Code to benchmark goes here + /// }), + /// ); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + pub fn bench_function<F>(&mut self, id: &str, f: F) -> &mut Criterion<M> + where + F: FnMut(&mut Bencher<'_, M>), + { + self.benchmark_group(id) + .bench_function(BenchmarkId::no_function(), f); + self + } + + /// Benchmarks a function with an input. For comparing multiple functions or multiple inputs, + /// see `benchmark_group`. + /// + /// # Example + /// + /// ```rust + /// #[macro_use] extern crate criterion; + /// use self::criterion::*; + /// + /// fn bench(c: &mut Criterion) { + /// // Setup (construct data, allocate memory, etc) + /// let input = 5u64; + /// c.bench_with_input( + /// BenchmarkId::new("function_name", input), &input, + /// |b, i| b.iter(|| { + /// // Code to benchmark using input `i` goes here + /// }), + /// ); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + pub fn bench_with_input<F, I>(&mut self, id: BenchmarkId, input: &I, f: F) -> &mut Criterion<M> + where + F: FnMut(&mut Bencher<'_, M>, &I), + { + // Guaranteed safe because external callers can't create benchmark IDs without a function + // name or parameter + let group_name = id.function_name.unwrap(); + let parameter = id.parameter.unwrap(); + self.benchmark_group(group_name).bench_with_input( + BenchmarkId::no_function_with_input(parameter), + input, + f, + ); + self + } + + /// Benchmarks a function under various inputs + /// + /// This is a convenience method to execute several related benchmarks. Each benchmark will + /// receive the id: `${id}/${input}`. + /// + /// # Example + /// + /// ```rust + /// # #[macro_use] extern crate criterion; + /// # use self::criterion::*; + /// + /// fn bench(c: &mut Criterion) { + /// c.bench_function_over_inputs("from_elem", + /// |b: &mut Bencher, size: &usize| { + /// b.iter(|| vec![0u8; *size]); + /// }, + /// vec![1024, 2048, 4096] + /// ); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + #[doc(hidden)] // Soft-deprecated, use benchmark groups instead + pub fn bench_function_over_inputs<I, F>( + &mut self, + id: &str, + f: F, + inputs: I, + ) -> &mut Criterion<M> + where + I: IntoIterator, + I::Item: fmt::Debug + 'static, + F: FnMut(&mut Bencher<'_, M>, &I::Item) + 'static, + { + self.bench(id, ParameterizedBenchmark::new(id, f, inputs)) + } + + /// Benchmarks multiple functions + /// + /// All functions get the same input and are compared with the other implementations. + /// Works similar to `bench_function`, but with multiple functions. + /// + /// # Example + /// + /// ``` rust + /// # #[macro_use] extern crate criterion; + /// # use self::criterion::*; + /// # fn seq_fib(i: &u32) {} + /// # fn par_fib(i: &u32) {} + /// + /// fn bench_seq_fib(b: &mut Bencher, i: &u32) { + /// b.iter(|| { + /// seq_fib(i); + /// }); + /// } + /// + /// fn bench_par_fib(b: &mut Bencher, i: &u32) { + /// b.iter(|| { + /// par_fib(i); + /// }); + /// } + /// + /// fn bench(c: &mut Criterion) { + /// let sequential_fib = Fun::new("Sequential", bench_seq_fib); + /// let parallel_fib = Fun::new("Parallel", bench_par_fib); + /// let funs = vec![sequential_fib, parallel_fib]; + /// + /// c.bench_functions("Fibonacci", funs, 14); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + #[doc(hidden)] // Soft-deprecated, use benchmark groups instead + pub fn bench_functions<I>( + &mut self, + id: &str, + funs: Vec<Fun<I, M>>, + input: I, + ) -> &mut Criterion<M> + where + I: fmt::Debug + 'static, + { + let benchmark = ParameterizedBenchmark::with_functions( + funs.into_iter().map(|fun| fun.f).collect(), + vec![input], + ); + + self.bench(id, benchmark) + } + + /// Executes the given benchmark. Use this variant to execute benchmarks + /// with complex configuration. This can be used to compare multiple + /// functions, execute benchmarks with custom configuration settings and + /// more. See the Benchmark and ParameterizedBenchmark structs for more + /// information. + /// + /// ```rust + /// # #[macro_use] extern crate criterion; + /// # use criterion::*; + /// # fn routine_1() {} + /// # fn routine_2() {} + /// + /// fn bench(c: &mut Criterion) { + /// // Setup (construct data, allocate memory, etc) + /// c.bench( + /// "routines", + /// Benchmark::new("routine_1", |b| b.iter(|| routine_1())) + /// .with_function("routine_2", |b| b.iter(|| routine_2())) + /// .sample_size(50) + /// ); + /// } + /// + /// criterion_group!(benches, bench); + /// criterion_main!(benches); + /// ``` + #[doc(hidden)] // Soft-deprecated, use benchmark groups instead + pub fn bench<B: BenchmarkDefinition<M>>( + &mut self, + group_id: &str, + benchmark: B, + ) -> &mut Criterion<M> { + benchmark.run(group_id, self); + self + } +} + +trait DurationExt { + fn to_nanos(&self) -> u64; +} + +const NANOS_PER_SEC: u64 = 1_000_000_000; + +impl DurationExt for Duration { + fn to_nanos(&self) -> u64 { + self.as_secs() * NANOS_PER_SEC + u64::from(self.subsec_nanos()) + } +} + +/// Enum representing different ways of measuring the throughput of benchmarked code. +/// If the throughput setting is configured for a benchmark then the estimated throughput will +/// be reported as well as the time per iteration. +// TODO: Remove serialize/deserialize from the public API. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum Throughput { + /// Measure throughput in terms of bytes/second. The value should be the number of bytes + /// processed by one iteration of the benchmarked code. Typically, this would be the length of + /// an input string or `&[u8]`. + Bytes(u64), + + /// Measure throughput in terms of elements/second. The value should be the number of elements + /// processed by one iteration of the benchmarked code. Typically, this would be the size of a + /// collection, but could also be the number of lines of input text or the number of values to + /// parse. + Elements(u64), +} + +/// Axis scaling type +#[derive(Debug, Clone, Copy)] +pub enum AxisScale { + /// Axes scale linearly + Linear, + + /// Axes scale logarithmically + Logarithmic, +} + +/// Contains the configuration options for the plots generated by a particular benchmark +/// or benchmark group. +/// +/// ```rust +/// use self::criterion::{Bencher, Criterion, Benchmark, PlotConfiguration, AxisScale}; +/// +/// let plot_config = PlotConfiguration::default() +/// .summary_scale(AxisScale::Logarithmic); +/// +/// // Using Criterion::default() for simplicity; normally you'd use the macros. +/// let mut criterion = Criterion::default(); +/// let mut benchmark_group = criterion.benchmark_group("Group name"); +/// benchmark_group.plot_config(plot_config); +/// // Use benchmark group +/// ``` +#[derive(Debug, Clone)] +pub struct PlotConfiguration { + summary_scale: AxisScale, +} + +impl Default for PlotConfiguration { + fn default() -> PlotConfiguration { + PlotConfiguration { + summary_scale: AxisScale::Linear, + } + } +} + +impl PlotConfiguration { + /// Set the axis scale (linear or logarithmic) for the summary plots. Typically, you would + /// set this to logarithmic if benchmarking over a range of inputs which scale exponentially. + /// Defaults to linear. + pub fn summary_scale(mut self, new_scale: AxisScale) -> PlotConfiguration { + self.summary_scale = new_scale; + self + } +} + +/// This enum allows the user to control how Criterion.rs chooses the iteration count when sampling. +/// The default is Auto, which will choose a method automatically based on the iteration time during +/// the warm-up phase. +#[derive(Debug, Clone, Copy)] +pub enum SamplingMode { + /// Criterion.rs should choose a sampling method automatically. This is the default, and is + /// recommended for most users and most benchmarks. + Auto, + + /// Scale the iteration count in each sample linearly. This is suitable for most benchmarks, + /// but it tends to require many iterations which can make it very slow for very long benchmarks. + Linear, + + /// Keep the iteration count the same for all samples. This is not recommended, as it affects + /// the statistics that Criterion.rs can compute. However, it requires fewer iterations than + /// the Linear method and therefore is more suitable for very long-running benchmarks where + /// benchmark execution time is more of a problem and statistical precision is less important. + Flat, +} +impl SamplingMode { + pub(crate) fn choose_sampling_mode( + &self, + warmup_mean_execution_time: f64, + sample_count: u64, + target_time: f64, + ) -> ActualSamplingMode { + match self { + SamplingMode::Linear => ActualSamplingMode::Linear, + SamplingMode::Flat => ActualSamplingMode::Flat, + SamplingMode::Auto => { + // Estimate execution time with linear sampling + let total_runs = sample_count * (sample_count + 1) / 2; + let d = + (target_time / warmup_mean_execution_time / total_runs as f64).ceil() as u64; + let expected_ns = total_runs as f64 * d as f64 * warmup_mean_execution_time; + + if expected_ns > (2.0 * target_time) { + ActualSamplingMode::Flat + } else { + ActualSamplingMode::Linear + } + } + } + } +} + +/// Enum to represent the sampling mode without Auto. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub(crate) enum ActualSamplingMode { + Linear, + Flat, +} +impl ActualSamplingMode { + pub(crate) fn iteration_counts( + &self, + warmup_mean_execution_time: f64, + sample_count: u64, + target_time: &Duration, + ) -> Vec<u64> { + match self { + ActualSamplingMode::Linear => { + let n = sample_count; + let met = warmup_mean_execution_time; + let m_ns = target_time.to_nanos(); + // Solve: [d + 2*d + 3*d + ... + n*d] * met = m_ns + let total_runs = n * (n + 1) / 2; + let d = (m_ns as f64 / met / total_runs as f64).ceil() as u64; + let expected_ns = total_runs as f64 * d as f64 * met; + + if d == 1 { + let recommended_sample_size = + ActualSamplingMode::recommend_linear_sample_size(m_ns as f64, met); + let actual_time = Duration::from_nanos(expected_ns as u64); + print!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}", + n, target_time, actual_time); + + if recommended_sample_size != n { + println!( + ", enable flat sampling, or reduce sample count to {}.", + recommended_sample_size + ); + } else { + println!("or enable flat sampling."); + } + } + + (1..(n + 1) as u64).map(|a| a * d).collect::<Vec<u64>>() + } + ActualSamplingMode::Flat => { + let n = sample_count; + let met = warmup_mean_execution_time; + let m_ns = target_time.to_nanos() as f64; + let time_per_sample = m_ns / (n as f64); + // This is pretty simplistic; we could do something smarter to fit into the allotted time. + let iterations_per_sample = (time_per_sample / met).ceil() as u64; + + let expected_ns = met * (iterations_per_sample * n) as f64; + + if iterations_per_sample == 1 { + let recommended_sample_size = + ActualSamplingMode::recommend_flat_sample_size(m_ns, met); + let actual_time = Duration::from_nanos(expected_ns as u64); + print!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}", + n, target_time, actual_time); + + if recommended_sample_size != n { + println!(", or reduce sample count to {}.", recommended_sample_size); + } else { + println!("."); + } + } + + vec![iterations_per_sample; n as usize] + } + } + } + + fn is_linear(&self) -> bool { + match self { + ActualSamplingMode::Linear => true, + _ => false, + } + } + + fn recommend_linear_sample_size(target_time: f64, met: f64) -> u64 { + // Some math shows that n(n+1)/2 * d * met = target_time. d = 1, so it can be ignored. + // This leaves n(n+1) = (2*target_time)/met, or n^2 + n - (2*target_time)/met = 0 + // Which can be solved with the quadratic formula. Since A and B are constant 1, + // this simplifies to sample_size = (-1 +- sqrt(1 - 4C))/2, where C = (2*target_time)/met. + // We don't care about the negative solution. Experimentation shows that this actually tends to + // result in twice the desired execution time (probably because of the ceil used to calculate + // d) so instead I use c = target_time/met. + let c = target_time / met; + let sample_size = (-1.0 + (4.0 * c).sqrt()) / 2.0; + let sample_size = sample_size as u64; + + // Round down to the nearest 10 to give a margin and avoid excessive precision + let sample_size = (sample_size / 10) * 10; + + // Clamp it to be at least 10, since criterion.rs doesn't allow sample sizes smaller than 10. + if sample_size < 10 { + 10 + } else { + sample_size + } + } + + fn recommend_flat_sample_size(target_time: f64, met: f64) -> u64 { + let sample_size = (target_time / met) as u64; + + // Round down to the nearest 10 to give a margin and avoid excessive precision + let sample_size = (sample_size / 10) * 10; + + // Clamp it to be at least 10, since criterion.rs doesn't allow sample sizes smaller than 10. + if sample_size < 10 { + 10 + } else { + sample_size + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct SavedSample { + sampling_mode: ActualSamplingMode, + iters: Vec<f64>, + times: Vec<f64>, +} + +/// Custom-test-framework runner. Should not be called directly. +#[doc(hidden)] +pub fn runner(benches: &[&dyn Fn()]) { + for bench in benches { + bench(); + } + Criterion::default().configure_from_args().final_summary(); +} diff --git a/src/macros.rs b/src/macros.rs new file mode 100755 index 0000000..ca8bb5c --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,132 @@ +//! Contains macros which together define a benchmark harness that can be used +//! in place of the standard benchmark harness. This allows the user to run +//! Criterion.rs benchmarks with `cargo bench`. + +/// Macro used to define a function group for the benchmark harness; see the +/// `criterion_main!` macro for more details. +/// +/// This is used to define a function group; a collection of functions to call with a common +/// Criterion configuration. Accepts two forms which can be seen below. +/// +/// Note that the group name given here is not important, it must simply +/// be unique. Note also that this macro is not related to the `Criterion::benchmark_group` function +/// or the `BenchmarkGroup` type. +/// +/// # Examples: +/// +/// Complete form: +/// +/// ``` +/// # #[macro_use] +/// # extern crate criterion; +/// # use criterion::Criterion; +/// # fn bench_method1(c: &mut Criterion) { +/// # } +/// # +/// # fn bench_method2(c: &mut Criterion) { +/// # } +/// # +/// criterion_group!{ +/// name = benches; +/// config = Criterion::default(); +/// targets = bench_method1, bench_method2 +/// } +/// # +/// # fn main() {} +/// ``` +/// +/// In this form, all of the options are clearly spelled out. This expands to +/// a function named benches, which uses the given config expression to create +/// an instance of the Criterion struct. This is then passed by mutable +/// reference to the targets. +/// +/// Compact Form: +/// +/// ``` +/// # #[macro_use] +/// # extern crate criterion; +/// # use criterion::Criterion; +/// # fn bench_method1(c: &mut Criterion) { +/// # } +/// # +/// # fn bench_method2(c: &mut Criterion) { +/// # } +/// # +/// criterion_group!(benches, bench_method1, bench_method2); +/// # +/// # fn main() {} +/// ``` +/// In this form, the first parameter is the name of the group and subsequent +/// parameters are the target methods. The Criterion struct will be created using +/// the `Criterion::default()` function. If you wish to customize the +/// configuration, use the complete form and provide your own configuration +/// function. +#[macro_export] +macro_rules! criterion_group { + (name = $name:ident; config = $config:expr; targets = $( $target:path ),+ $(,)*) => { + pub fn $name() { + let mut criterion: $crate::Criterion<_> = $config + .configure_from_args(); + $( + $target(&mut criterion); + )+ + } + }; + ($name:ident, $( $target:path ),+ $(,)*) => { + criterion_group!{ + name = $name; + config = $crate::Criterion::default(); + targets = $( $target ),+ + } + } +} + +/// Macro which expands to a benchmark harness. +/// +/// Currently, using Criterion.rs requires disabling the benchmark harness +/// generated automatically by rustc. This can be done like so: +/// +/// ```toml +/// [[bench]] +/// name = "my_bench" +/// harness = false +/// ``` +/// +/// In this case, `my_bench` must be a rust file inside the 'benches' directory, +/// like so: +/// +/// `benches/my_bench.rs` +/// +/// Since we've disabled the default benchmark harness, we need to add our own: +/// +/// ```ignore +/// #[macro_use] +/// extern crate criterion; +/// use criterion::Criterion; +/// fn bench_method1(c: &mut Criterion) { +/// } +/// +/// fn bench_method2(c: &mut Criterion) { +/// } +/// +/// criterion_group!(benches, bench_method1, bench_method2); +/// criterion_main!(benches); +/// ``` +/// +/// The `criterion_main` macro expands to a `main` function which runs all of the +/// benchmarks in the given groups. +/// +#[macro_export] +macro_rules! criterion_main { + ( $( $group:path ),+ $(,)* ) => { + fn main() { + $( + $group(); + )+ + + $crate::Criterion::default() + .configure_from_args() + .final_summary(); + } + } +} diff --git a/src/macros_private.rs b/src/macros_private.rs new file mode 100755 index 0000000..982602f --- /dev/null +++ b/src/macros_private.rs @@ -0,0 +1,47 @@ +//! Private macro used for error handling. + +/// Logs an error, ignores an `Ok` value. +macro_rules! log_if_err { + ($x:expr) => { + let closure = || { + try_else_return!($x); + }; + closure(); + }; +} + +/// Matches a result, returning the `Ok` value in case of success, +/// exits the calling function otherwise. +/// A closure which returns the return value for the function can +/// be passed as second parameter. +macro_rules! try_else_return { + ($x:expr) => { + try_else_return!($x, || {}); + }; + ($x:expr, $el:expr) => { + match $x { + Ok(x) => x, + Err(e) => { + crate::error::log_error(&e); + let closure = $el; + return closure(); + } + } + }; +} + +/// Print an error message to stdout. Format is the same as println! or format! +macro_rules! error { + ($($arg:tt)*) => ( + println!("Criterion.rs ERROR: {}", &format!($($arg)*)); + ) +} + +/// Print a debug message to stdout. Format is the same as println! or format! +macro_rules! info { + ($($arg:tt)*) => ( + if $crate::debug_enabled() { + println!("Criterion.rs DEBUG: {}", &format!($($arg)*)); + } + ) +} diff --git a/src/measurement.rs b/src/measurement.rs new file mode 100755 index 0000000..8c6d708 --- /dev/null +++ b/src/measurement.rs @@ -0,0 +1,212 @@ +//! This module defines a set of traits that can be used to plug different measurements (eg. +//! Unix's Processor Time, CPU or GPU performance counters, etc.) into Criterion.rs. It also +//! includes the [WallTime](struct.WallTime.html) struct which defines the default wall-clock time +//! measurement. + +use crate::format::short; +use crate::DurationExt; +use crate::Throughput; +use std::time::{Duration, Instant}; + +/// Trait providing functions to format measured values to string so that they can be displayed on +/// the command line or in the reports. The functions of this trait take measured values in f64 +/// form; implementors can assume that the values are of the same scale as those produced by the +/// associated [MeasuredValue](trait.MeasuredValue.html) (eg. if your measurement produces values in +/// nanoseconds, the values passed to the formatter will be in nanoseconds). +/// +/// Implementors are encouraged to format the values in a way that is intuitive for humans and +/// uses the SI prefix system. For example, the format used by [WallTime](struct.Walltime.html) +/// can display the value in units ranging from picoseconds to seconds depending on the magnitude +/// of the elapsed time in nanoseconds. +pub trait ValueFormatter { + /// Format the value (with appropriate unit) and return it as a string. + fn format_value(&self, value: f64) -> String { + let mut values = [value]; + let unit = self.scale_values(value, &mut values); + format!("{:>6} {}", short(values[0]), unit) + } + + /// Format the value as a throughput measurement. The value represents the measurement value; + /// the implementor will have to calculate bytes per second, iterations per cycle, etc. + fn format_throughput(&self, throughput: &Throughput, value: f64) -> String { + let mut values = [value]; + let unit = self.scale_throughputs(value, throughput, &mut values); + format!("{:>6} {}", short(values[0]), unit) + } + + /// Scale the given values to some appropriate unit and return the unit string. + /// + /// The given typical value should be used to choose the unit. This function may be called + /// multiple times with different datasets; the typical value will remain the same to ensure + /// that the units remain consistent within a graph. The typical value will not be NaN. + /// Values will not contain NaN as input, and the transformed values must not contain NaN. + fn scale_values(&self, typical_value: f64, values: &mut [f64]) -> &'static str; + + /// Convert the given measured values into throughput numbers based on the given throughput + /// value, scale them to some appropriate unit, and return the unit string. + /// + /// The given typical value should be used to choose the unit. This function may be called + /// multiple times with different datasets; the typical value will remain the same to ensure + /// that the units remain consistent within a graph. The typical value will not be NaN. + /// Values will not contain NaN as input, and the transformed values must not contain NaN. + fn scale_throughputs( + &self, + typical_value: f64, + throughput: &Throughput, + values: &mut [f64], + ) -> &'static str; + + /// Scale the values and return a unit string designed for machines. + /// + /// For example, this is used for the CSV file output. Implementations should modify the given + /// values slice to apply the desired scaling (if any) and return a string representing the unit + /// the modified values are in. + fn scale_for_machines(&self, values: &mut [f64]) -> &'static str; +} + +/// Trait for all types which define something Criterion.rs can measure. The only measurement +/// currently provided is [WallTime](struct.WallTime.html), but third party crates or benchmarks +/// may define more. +/// +/// This trait defines two core methods, `start` and `end`. `start` is called at the beginning of +/// a measurement to produce some intermediate value (for example, the wall-clock time at the start +/// of that set of iterations) and `end` is called at the end of the measurement with the value +/// returned by `start`. +/// +pub trait Measurement { + /// This type represents an intermediate value for the measurements. It will be produced by the + /// start function and passed to the end function. An example might be the wall-clock time as + /// of the `start` call. + type Intermediate; + + /// This type is the measured value. An example might be the elapsed wall-clock time between the + /// `start` and `end` calls. + type Value; + + /// Criterion.rs will call this before iterating the benchmark. + fn start(&self) -> Self::Intermediate; + + /// Criterion.rs will call this after iterating the benchmark to get the measured value. + fn end(&self, i: Self::Intermediate) -> Self::Value; + + /// Combine two values. Criterion.rs sometimes needs to perform measurements in multiple batches + /// of iterations, so the value from one batch must be added to the sum of the previous batches. + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value; + + /// Return a "zero" value for the Value type which can be added to another value. + fn zero(&self) -> Self::Value; + + /// Converts the measured value to f64 so that it can be used in statistical analysis. + fn to_f64(&self, value: &Self::Value) -> f64; + + /// Return a trait-object reference to the value formatter for this measurement. + fn formatter(&self) -> &dyn ValueFormatter; +} + +pub(crate) struct DurationFormatter; +impl DurationFormatter { + fn bytes_per_second(&self, bytes: f64, typical: f64, values: &mut [f64]) -> &'static str { + let bytes_per_second = bytes * (1e9 / typical); + let (denominator, unit) = if bytes_per_second < 1024.0 { + (1.0, " B/s") + } else if bytes_per_second < 1024.0 * 1024.0 { + (1024.0, "KiB/s") + } else if bytes_per_second < 1024.0 * 1024.0 * 1024.0 { + (1024.0 * 1024.0, "MiB/s") + } else { + (1024.0 * 1024.0 * 1024.0, "GiB/s") + }; + + for val in values { + let bytes_per_second = bytes * (1e9 / *val); + *val = bytes_per_second / denominator; + } + + unit + } + + fn elements_per_second(&self, elems: f64, typical: f64, values: &mut [f64]) -> &'static str { + let elems_per_second = elems * (1e9 / typical); + let (denominator, unit) = if elems_per_second < 1000.0 { + (1.0, " elem/s") + } else if elems_per_second < 1000.0 * 1000.0 { + (1000.0, "Kelem/s") + } else if elems_per_second < 1000.0 * 1000.0 * 1000.0 { + (1000.0 * 1000.0, "Melem/s") + } else { + (1000.0 * 1000.0 * 1000.0, "Gelem/s") + }; + + for val in values { + let elems_per_second = elems * (1e9 / *val); + *val = elems_per_second / denominator; + } + + unit + } +} +impl ValueFormatter for DurationFormatter { + fn scale_throughputs( + &self, + typical: f64, + throughput: &Throughput, + values: &mut [f64], + ) -> &'static str { + match *throughput { + Throughput::Bytes(bytes) => self.bytes_per_second(bytes as f64, typical, values), + Throughput::Elements(elems) => self.elements_per_second(elems as f64, typical, values), + } + } + + fn scale_values(&self, ns: f64, values: &mut [f64]) -> &'static str { + let (factor, unit) = if ns < 10f64.powi(0) { + (10f64.powi(3), "ps") + } else if ns < 10f64.powi(3) { + (10f64.powi(0), "ns") + } else if ns < 10f64.powi(6) { + (10f64.powi(-3), "us") + } else if ns < 10f64.powi(9) { + (10f64.powi(-6), "ms") + } else { + (10f64.powi(-9), "s") + }; + + for val in values { + *val *= factor; + } + + unit + } + + fn scale_for_machines(&self, _values: &mut [f64]) -> &'static str { + // no scaling is needed + "ns" + } +} + +/// `WallTime` is the default measurement in Criterion.rs. It measures the elapsed time from the +/// beginning of a series of iterations to the end. +pub struct WallTime; +impl Measurement for WallTime { + type Intermediate = Instant; + type Value = Duration; + + fn start(&self) -> Self::Intermediate { + Instant::now() + } + fn end(&self, i: Self::Intermediate) -> Self::Value { + i.elapsed() + } + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { + *v1 + *v2 + } + fn zero(&self) -> Self::Value { + Duration::from_secs(0) + } + fn to_f64(&self, val: &Self::Value) -> f64 { + val.to_nanos() as f64 + } + fn formatter(&self) -> &dyn ValueFormatter { + &DurationFormatter + } +} diff --git a/src/plot/gnuplot_backend/distributions.rs b/src/plot/gnuplot_backend/distributions.rs new file mode 100755 index 0000000..1ccbc1a --- /dev/null +++ b/src/plot/gnuplot_backend/distributions.rs @@ -0,0 +1,310 @@ +use std::iter; +use std::process::Child; + +use crate::stats::univariate::Sample; +use crate::stats::Distribution; +use criterion_plot::prelude::*; + +use super::*; +use crate::estimate::Estimate; +use crate::estimate::Statistic; +use crate::kde; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; + +fn abs_distribution( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + statistic: Statistic, + distribution: &Distribution<f64>, + estimate: &Estimate, + size: Option<Size>, +) -> Child { + let ci = &estimate.confidence_interval; + let typical = ci.upper_bound; + let mut ci_values = [ci.lower_bound, ci.upper_bound, estimate.point_estimate]; + let unit = formatter.scale_values(typical, &mut ci_values); + let (lb, ub, point) = (ci_values[0], ci_values[1], ci_values[2]); + + let start = lb - (ub - lb) / 9.; + let end = ub + (ub - lb) / 9.; + let mut scaled_xs: Vec<f64> = distribution.iter().cloned().collect(); + let _ = formatter.scale_values(typical, &mut scaled_xs); + let scaled_xs_sample = Sample::new(&scaled_xs); + let (kde_xs, ys) = kde::sweep(scaled_xs_sample, KDE_POINTS, Some((start, end))); + + // interpolate between two points of the KDE sweep to find the Y position at the point estimate. + let n_point = kde_xs + .iter() + .position(|&x| x >= point) + .unwrap_or(kde_xs.len() - 1) + .max(1); // Must be at least the second element or this will panic + let slope = (ys[n_point] - ys[n_point - 1]) / (kde_xs[n_point] - kde_xs[n_point - 1]); + let y_point = ys[n_point - 1] + (slope * (point - kde_xs[n_point - 1])); + + let zero = iter::repeat(0); + + let start = kde_xs + .iter() + .enumerate() + .find(|&(_, &x)| x >= lb) + .unwrap() + .0; + let end = kde_xs + .iter() + .enumerate() + .rev() + .find(|&(_, &x)| x <= ub) + .unwrap() + .0; + let len = end - start; + + let kde_xs_sample = Sample::new(&kde_xs); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .set(Title(format!( + "{}: {}", + gnuplot_escape(id.as_title()), + statistic + ))) + .configure(Axis::BottomX, |a| { + a.set(Label(format!("Average time ({})", unit))) + .set(Range::Limits(kde_xs_sample.min(), kde_xs_sample.max())) + }) + .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)"))) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Outside(Vertical::Top, Horizontal::Right)) + }) + .plot( + Lines { + x: &*kde_xs, + y: &*ys, + }, + |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("Bootstrap distribution")) + .set(LineType::Solid) + }, + ) + .plot( + FilledCurve { + x: kde_xs.iter().skip(start).take(len), + y1: ys.iter().skip(start), + y2: zero, + }, + |c| { + c.set(DARK_BLUE) + .set(Label("Confidence interval")) + .set(Opacity(0.25)) + }, + ) + .plot( + Lines { + x: &[point, point], + y: &[0., y_point], + }, + |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("Point estimate")) + .set(LineType::Dash) + }, + ); + + let path = context.report_path(id, &format!("{}.svg", statistic)); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn abs_distributions( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Vec<Child> { + crate::plot::REPORT_STATS + .iter() + .filter_map(|stat| { + measurements.distributions.get(*stat).and_then(|dist| { + measurements + .absolute_estimates + .get(*stat) + .map(|est| (*stat, dist, est)) + }) + }) + .map(|(statistic, distribution, estimate)| { + abs_distribution( + id, + context, + formatter, + statistic, + distribution, + estimate, + size, + ) + }) + .collect::<Vec<_>>() +} + +fn rel_distribution( + id: &BenchmarkId, + context: &ReportContext, + statistic: Statistic, + distribution: &Distribution<f64>, + estimate: &Estimate, + noise_threshold: f64, + size: Option<Size>, +) -> Child { + let ci = &estimate.confidence_interval; + let (lb, ub) = (ci.lower_bound, ci.upper_bound); + + let start = lb - (ub - lb) / 9.; + let end = ub + (ub - lb) / 9.; + let (xs, ys) = kde::sweep(distribution, KDE_POINTS, Some((start, end))); + let xs_ = Sample::new(&xs); + + // interpolate between two points of the KDE sweep to find the Y position at the point estimate. + let point = estimate.point_estimate; + let n_point = xs + .iter() + .position(|&x| x >= point) + .unwrap_or(ys.len() - 1) + .max(1); + let slope = (ys[n_point] - ys[n_point - 1]) / (xs[n_point] - xs[n_point - 1]); + let y_point = ys[n_point - 1] + (slope * (point - xs[n_point - 1])); + + let one = iter::repeat(1); + let zero = iter::repeat(0); + + let start = xs.iter().enumerate().find(|&(_, &x)| x >= lb).unwrap().0; + let end = xs + .iter() + .enumerate() + .rev() + .find(|&(_, &x)| x <= ub) + .unwrap() + .0; + let len = end - start; + + let x_min = xs_.min(); + let x_max = xs_.max(); + + let (fc_start, fc_end) = if noise_threshold < x_min || -noise_threshold > x_max { + let middle = (x_min + x_max) / 2.; + + (middle, middle) + } else { + ( + if -noise_threshold < x_min { + x_min + } else { + -noise_threshold + }, + if noise_threshold > x_max { + x_max + } else { + noise_threshold + }, + ) + }; + + let mut figure = Figure::new(); + + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)"))) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Outside(Vertical::Top, Horizontal::Right)) + }) + .set(Title(format!( + "{}: {}", + gnuplot_escape(id.as_title()), + statistic + ))) + .configure(Axis::BottomX, |a| { + a.set(Label("Relative change (%)")) + .set(Range::Limits(x_min * 100., x_max * 100.)) + .set(ScaleFactor(100.)) + }) + .plot(Lines { x: &*xs, y: &*ys }, |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("Bootstrap distribution")) + .set(LineType::Solid) + }) + .plot( + FilledCurve { + x: xs.iter().skip(start).take(len), + y1: ys.iter().skip(start), + y2: zero.clone(), + }, + |c| { + c.set(DARK_BLUE) + .set(Label("Confidence interval")) + .set(Opacity(0.25)) + }, + ) + .plot( + Lines { + x: &[point, point], + y: &[0., y_point], + }, + |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("Point estimate")) + .set(LineType::Dash) + }, + ) + .plot( + FilledCurve { + x: &[fc_start, fc_end], + y1: one, + y2: zero, + }, + |c| { + c.set(Axes::BottomXRightY) + .set(DARK_RED) + .set(Label("Noise threshold")) + .set(Opacity(0.1)) + }, + ); + + let path = context.report_path(id, &format!("change/{}.svg", statistic)); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn rel_distributions( + id: &BenchmarkId, + context: &ReportContext, + _measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Vec<Child> { + crate::plot::CHANGE_STATS + .iter() + .map(|&statistic| { + rel_distribution( + id, + context, + statistic, + comparison.relative_distributions.get(statistic), + comparison.relative_estimates.get(statistic), + comparison.noise_threshold, + size, + ) + }) + .collect::<Vec<_>>() +} diff --git a/src/plot/gnuplot_backend/iteration_times.rs b/src/plot/gnuplot_backend/iteration_times.rs new file mode 100755 index 0000000..4db4de8 --- /dev/null +++ b/src/plot/gnuplot_backend/iteration_times.rs @@ -0,0 +1,173 @@ +use std::process::Child; + +use criterion_plot::prelude::*; + +use super::*; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; + +use crate::measurement::ValueFormatter; + +fn iteration_times_figure( + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Figure { + let data = &measurements.avg_times; + let max_avg_time = data.max(); + let mut scaled_y: Vec<_> = data.iter().map(|(f, _)| f).collect(); + let unit = formatter.scale_values(max_avg_time, &mut scaled_y); + let scaled_y = Sample::new(&scaled_y); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + a.configure(Grid::Major, |g| g.show()).set(Label("Sample")) + }) + .configure(Axis::LeftY, |a| { + a.configure(Grid::Major, |g| g.show()) + .set(Label(format!("Average Iteration Time ({})", unit))) + }) + .plot( + Points { + x: 1..(data.len() + 1), + y: scaled_y.as_ref(), + }, + |c| { + c.set(DARK_BLUE) + .set(PointSize(0.5)) + .set(PointType::FilledCircle) + }, + ); + figure +} + +pub(crate) fn iteration_times( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Child { + let mut figure = iteration_times_figure(formatter, measurements, size); + figure.set(Title(gnuplot_escape(id.as_title()))); + figure.configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Inside(Vertical::Top, Horizontal::Left)) + }); + + let path = context.report_path(id, "iteration_times.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn iteration_times_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Child { + let mut figure = iteration_times_figure(formatter, measurements, size); + figure.configure(Key, |k| k.hide()); + + let path = context.report_path(id, "iteration_times_small.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +fn iteration_times_comparison_figure( + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Figure { + let current_data = &measurements.avg_times; + let base_data = &comparison.base_avg_times; + + let mut all_data: Vec<f64> = current_data.iter().map(|(f, _)| f).collect(); + all_data.extend_from_slice(base_data); + + let typical_value = Sample::new(&all_data).max(); + let unit = formatter.scale_values(typical_value, &mut all_data); + + let (scaled_current_y, scaled_base_y) = all_data.split_at(current_data.len()); + let scaled_current_y = Sample::new(scaled_current_y); + let scaled_base_y = Sample::new(scaled_base_y); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + a.configure(Grid::Major, |g| g.show()).set(Label("Sample")) + }) + .configure(Axis::LeftY, |a| { + a.configure(Grid::Major, |g| g.show()) + .set(Label(format!("Average Iteration Time ({})", unit))) + }) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Inside(Vertical::Top, Horizontal::Left)) + }) + .plot( + Points { + x: 1..(current_data.len() + 1), + y: scaled_base_y.as_ref(), + }, + |c| { + c.set(DARK_RED) + .set(Label("Base")) + .set(PointSize(0.5)) + .set(PointType::FilledCircle) + }, + ) + .plot( + Points { + x: 1..(current_data.len() + 1), + y: scaled_current_y.as_ref(), + }, + |c| { + c.set(DARK_BLUE) + .set(Label("Current")) + .set(PointSize(0.5)) + .set(PointType::FilledCircle) + }, + ); + figure +} + +pub(crate) fn iteration_times_comparison( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Child { + let mut figure = iteration_times_comparison_figure(formatter, measurements, comparison, size); + figure.set(Title(gnuplot_escape(id.as_title()))); + + let path = context.report_path(id, "both/iteration_times.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn iteration_times_comparison_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Child { + let mut figure = iteration_times_comparison_figure(formatter, measurements, comparison, size); + figure.configure(Key, |k| k.hide()); + + let path = context.report_path(id, "relative_iteration_times_small.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} diff --git a/src/plot/gnuplot_backend/mod.rs b/src/plot/gnuplot_backend/mod.rs new file mode 100755 index 0000000..e902d47 --- /dev/null +++ b/src/plot/gnuplot_backend/mod.rs @@ -0,0 +1,254 @@ +use std::iter; +use std::path::PathBuf; +use std::process::Child; + +use crate::stats::univariate::Sample; +use criterion_plot::prelude::*; + +mod distributions; +mod iteration_times; +mod pdf; +mod regression; +mod summary; +mod t_test; +use self::distributions::*; +use self::iteration_times::*; +use self::pdf::*; +use self::regression::*; +use self::summary::*; +use self::t_test::*; + +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ValueType}; +use crate::stats::bivariate::Data; + +use super::{PlotContext, PlotData, Plotter}; +use crate::format; + +fn gnuplot_escape(string: &str) -> String { + string.replace("_", "\\_").replace("'", "''") +} + +static DEFAULT_FONT: &str = "Helvetica"; +static KDE_POINTS: usize = 500; +static SIZE: Size = Size(1280, 720); + +const LINEWIDTH: LineWidth = LineWidth(2.); +const POINT_SIZE: PointSize = PointSize(0.75); + +const DARK_BLUE: Color = Color::Rgb(31, 120, 180); +const DARK_ORANGE: Color = Color::Rgb(255, 127, 0); +const DARK_RED: Color = Color::Rgb(227, 26, 28); + +fn debug_script(path: &PathBuf, figure: &Figure) { + if crate::debug_enabled() { + let mut script_path = path.clone(); + script_path.set_extension("gnuplot"); + info!("Writing gnuplot script to {:?}", script_path); + let result = figure.save(script_path.as_path()); + if let Err(e) = result { + error!("Failed to write debug output: {}", e); + } + } +} + +/// Private +trait Append<T> { + /// Private + fn append_(self, item: T) -> Self; +} + +// NB I wish this was in the standard library +impl<T> Append<T> for Vec<T> { + fn append_(mut self, item: T) -> Vec<T> { + self.push(item); + self + } +} + +#[derive(Default)] +pub(crate) struct Gnuplot { + process_list: Vec<Child>, +} + +impl Plotter for Gnuplot { + fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.push(if ctx.is_thumbnail { + if let Some(cmp) = data.comparison { + pdf_comparison_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + size, + ) + } else { + pdf_small(ctx.id, ctx.context, data.formatter, data.measurements, size) + } + } else if let Some(cmp) = data.comparison { + pdf_comparison( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + size, + ) + } else { + pdf(ctx.id, ctx.context, data.formatter, data.measurements, size) + }); + } + + fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.push(if ctx.is_thumbnail { + if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + regression_comparison_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + &base_data, + size, + ) + } else { + regression_small(ctx.id, ctx.context, data.formatter, data.measurements, size) + } + } else if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + regression_comparison( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + &base_data, + size, + ) + } else { + regression(ctx.id, ctx.context, data.formatter, data.measurements, size) + }); + } + + fn iteration_times(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.push(if ctx.is_thumbnail { + if let Some(cmp) = data.comparison { + iteration_times_comparison_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + size, + ) + } else { + iteration_times_small(ctx.id, ctx.context, data.formatter, data.measurements, size) + } + } else if let Some(cmp) = data.comparison { + iteration_times_comparison( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + size, + ) + } else { + iteration_times(ctx.id, ctx.context, data.formatter, data.measurements, size) + }); + } + + fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.extend(abs_distributions( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + size, + )); + } + + fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + if let Some(cmp) = data.comparison { + self.process_list.extend(rel_distributions( + ctx.id, + ctx.context, + data.measurements, + cmp, + size, + )); + } else { + error!("Comparison data is not provided for a relative distribution figure"); + } + } + + fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + if let Some(cmp) = data.comparison { + self.process_list + .push(t_test(ctx.id, ctx.context, data.measurements, cmp, size)); + } else { + error!("Comparison data is not provided for t_test plot"); + } + } + + fn line_comparison( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + value_type: ValueType, + ) { + let path = ctx.line_comparison_path(); + self.process_list.push(line_comparison( + formatter, + ctx.id.as_title(), + all_curves, + &path, + value_type, + ctx.context.plot_config.summary_scale, + )); + } + + fn violin( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + ) { + let violin_path = ctx.violin_path(); + + self.process_list.push(violin( + formatter, + ctx.id.as_title(), + all_curves, + &violin_path, + ctx.context.plot_config.summary_scale, + )); + } + + fn wait(&mut self) { + let start = std::time::Instant::now(); + let child_count = self.process_list.len(); + for child in self.process_list.drain(..) { + match child.wait_with_output() { + Ok(ref out) if out.status.success() => {} + Ok(out) => error!("Error in Gnuplot: {}", String::from_utf8_lossy(&out.stderr)), + Err(e) => error!("Got IO error while waiting for Gnuplot to complete: {}", e), + } + } + let elapsed = &start.elapsed(); + info!( + "Waiting for {} gnuplot processes took {}", + child_count, + format::time(crate::DurationExt::to_nanos(elapsed) as f64) + ); + } +} diff --git a/src/plot/gnuplot_backend/pdf.rs b/src/plot/gnuplot_backend/pdf.rs new file mode 100755 index 0000000..3ee2360 --- /dev/null +++ b/src/plot/gnuplot_backend/pdf.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::kde; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; +use std::process::Child; + +pub(crate) fn pdf( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Child { + let avg_times = &measurements.avg_times; + let typical = avg_times.max(); + let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_avg_times); + let scaled_avg_times = Sample::new(&scaled_avg_times); + + let mean = scaled_avg_times.mean(); + + let iter_counts = measurements.iter_counts(); + let &max_iters = iter_counts + .iter() + .max_by_key(|&&iters| iters as u64) + .unwrap(); + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let y_scale = 10f64.powi(-exponent); + + let y_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let (xs, ys) = kde::sweep(&scaled_avg_times, KDE_POINTS, None); + let (lost, lomt, himt, hist) = avg_times.fences(); + let mut fences = [lost, lomt, himt, hist]; + let _ = formatter.scale_values(typical, &mut fences); + let [lost, lomt, himt, hist] = fences; + + let vertical = &[0., max_iters]; + let zeros = iter::repeat(0); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + let xs_ = Sample::new(&xs); + a.set(Label(format!("Average time ({})", unit))) + .set(Range::Limits(xs_.min(), xs_.max())) + }) + .configure(Axis::LeftY, |a| { + a.set(Label(y_label)) + .set(Range::Limits(0., max_iters * y_scale)) + .set(ScaleFactor(y_scale)) + }) + .configure(Axis::RightY, |a| a.set(Label("Density (a.u.)"))) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Outside(Vertical::Top, Horizontal::Right)) + }) + .plot( + FilledCurve { + x: &*xs, + y1: &*ys, + y2: zeros, + }, + |c| { + c.set(Axes::BottomXRightY) + .set(DARK_BLUE) + .set(Label("PDF")) + .set(Opacity(0.25)) + }, + ) + .plot( + Lines { + x: &[mean, mean], + y: vertical, + }, + |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(LineType::Dash) + .set(Label("Mean")) + }, + ) + .plot( + Points { + x: avg_times + .iter() + .zip(scaled_avg_times.iter()) + .filter_map( + |((_, label), t)| { + if label.is_outlier() { + None + } else { + Some(t) + } + }, + ), + y: avg_times + .iter() + .zip(iter_counts.iter()) + .filter_map( + |((_, label), i)| { + if label.is_outlier() { + None + } else { + Some(i) + } + }, + ), + }, + |c| { + c.set(DARK_BLUE) + .set(Label("\"Clean\" sample")) + .set(PointType::FilledCircle) + .set(POINT_SIZE) + }, + ) + .plot( + Points { + x: avg_times + .iter() + .zip(scaled_avg_times.iter()) + .filter_map( + |((_, label), t)| { + if label.is_mild() { + Some(t) + } else { + None + } + }, + ), + y: avg_times + .iter() + .zip(iter_counts.iter()) + .filter_map( + |((_, label), i)| { + if label.is_mild() { + Some(i) + } else { + None + } + }, + ), + }, + |c| { + c.set(DARK_ORANGE) + .set(Label("Mild outliers")) + .set(POINT_SIZE) + .set(PointType::FilledCircle) + }, + ) + .plot( + Points { + x: avg_times + .iter() + .zip(scaled_avg_times.iter()) + .filter_map( + |((_, label), t)| { + if label.is_severe() { + Some(t) + } else { + None + } + }, + ), + y: avg_times + .iter() + .zip(iter_counts.iter()) + .filter_map( + |((_, label), i)| { + if label.is_severe() { + Some(i) + } else { + None + } + }, + ), + }, + |c| { + c.set(DARK_RED) + .set(Label("Severe outliers")) + .set(POINT_SIZE) + .set(PointType::FilledCircle) + }, + ) + .plot( + Lines { + x: &[lomt, lomt], + y: vertical, + }, + |c| c.set(DARK_ORANGE).set(LINEWIDTH).set(LineType::Dash), + ) + .plot( + Lines { + x: &[himt, himt], + y: vertical, + }, + |c| c.set(DARK_ORANGE).set(LINEWIDTH).set(LineType::Dash), + ) + .plot( + Lines { + x: &[lost, lost], + y: vertical, + }, + |c| c.set(DARK_RED).set(LINEWIDTH).set(LineType::Dash), + ) + .plot( + Lines { + x: &[hist, hist], + y: vertical, + }, + |c| c.set(DARK_RED).set(LINEWIDTH).set(LineType::Dash), + ); + figure.set(Title(gnuplot_escape(id.as_title()))); + + let path = context.report_path(id, "pdf.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn pdf_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Child { + let avg_times = &*measurements.avg_times; + let typical = avg_times.max(); + let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_avg_times); + let scaled_avg_times = Sample::new(&scaled_avg_times); + let mean = scaled_avg_times.mean(); + + let (xs, ys, mean_y) = kde::sweep_and_estimate(scaled_avg_times, KDE_POINTS, None, mean); + let xs_ = Sample::new(&xs); + let ys_ = Sample::new(&ys); + + let y_limit = ys_.max() * 1.1; + let zeros = iter::repeat(0); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + a.set(Label(format!("Average time ({})", unit))) + .set(Range::Limits(xs_.min(), xs_.max())) + }) + .configure(Axis::LeftY, |a| { + a.set(Label("Density (a.u.)")) + .set(Range::Limits(0., y_limit)) + }) + .configure(Axis::RightY, |a| a.hide()) + .configure(Key, |k| k.hide()) + .plot( + FilledCurve { + x: &*xs, + y1: &*ys, + y2: zeros, + }, + |c| { + c.set(Axes::BottomXRightY) + .set(DARK_BLUE) + .set(Label("PDF")) + .set(Opacity(0.25)) + }, + ) + .plot( + Lines { + x: &[mean, mean], + y: &[0., mean_y], + }, + |c| c.set(DARK_BLUE).set(LINEWIDTH).set(Label("Mean")), + ); + + let path = context.report_path(id, "pdf_small.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +fn pdf_comparison_figure( + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Figure { + let base_avg_times = Sample::new(&comparison.base_avg_times); + let typical = base_avg_times.max().max(measurements.avg_times.max()); + let mut scaled_base_avg_times: Vec<f64> = comparison.base_avg_times.clone(); + let unit = formatter.scale_values(typical, &mut scaled_base_avg_times); + let scaled_base_avg_times = Sample::new(&scaled_base_avg_times); + + let mut scaled_new_avg_times: Vec<f64> = (&measurements.avg_times as &Sample<f64>) + .iter() + .cloned() + .collect(); + let _ = formatter.scale_values(typical, &mut scaled_new_avg_times); + let scaled_new_avg_times = Sample::new(&scaled_new_avg_times); + + let base_mean = scaled_base_avg_times.mean(); + let new_mean = scaled_new_avg_times.mean(); + + let (base_xs, base_ys, base_y_mean) = + kde::sweep_and_estimate(scaled_base_avg_times, KDE_POINTS, None, base_mean); + let (xs, ys, y_mean) = + kde::sweep_and_estimate(scaled_new_avg_times, KDE_POINTS, None, new_mean); + + let zeros = iter::repeat(0); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + a.set(Label(format!("Average time ({})", unit))) + }) + .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)"))) + .configure(Axis::RightY, |a| a.hide()) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Outside(Vertical::Top, Horizontal::Right)) + }) + .plot( + FilledCurve { + x: &*base_xs, + y1: &*base_ys, + y2: zeros.clone(), + }, + |c| c.set(DARK_RED).set(Label("Base PDF")).set(Opacity(0.5)), + ) + .plot( + Lines { + x: &[base_mean, base_mean], + y: &[0., base_y_mean], + }, + |c| c.set(DARK_RED).set(Label("Base Mean")).set(LINEWIDTH), + ) + .plot( + FilledCurve { + x: &*xs, + y1: &*ys, + y2: zeros, + }, + |c| c.set(DARK_BLUE).set(Label("New PDF")).set(Opacity(0.5)), + ) + .plot( + Lines { + x: &[new_mean, new_mean], + y: &[0., y_mean], + }, + |c| c.set(DARK_BLUE).set(Label("New Mean")).set(LINEWIDTH), + ); + figure +} + +pub(crate) fn pdf_comparison( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Child { + let mut figure = pdf_comparison_figure(formatter, measurements, comparison, size); + figure.set(Title(gnuplot_escape(id.as_title()))); + let path = context.report_path(id, "both/pdf.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn pdf_comparison_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Child { + let mut figure = pdf_comparison_figure(formatter, measurements, comparison, size); + figure.configure(Key, |k| k.hide()); + let path = context.report_path(id, "relative_pdf_small.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} diff --git a/src/plot/gnuplot_backend/regression.rs b/src/plot/gnuplot_backend/regression.rs new file mode 100755 index 0000000..82de357 --- /dev/null +++ b/src/plot/gnuplot_backend/regression.rs @@ -0,0 +1,279 @@ +use std::process::Child; + +use crate::stats::bivariate::regression::Slope; +use criterion_plot::prelude::*; + +use super::*; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; +use crate::stats::bivariate::Data; + +use crate::estimate::{ConfidenceInterval, Estimate}; + +use crate::measurement::ValueFormatter; + +fn regression_figure( + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Figure { + let slope_estimate = measurements.absolute_estimates.slope.as_ref().unwrap(); + let slope_dist = measurements.distributions.slope.as_ref().unwrap(); + let (lb, ub) = + slope_dist.confidence_interval(slope_estimate.confidence_interval.confidence_level); + + let data = &measurements.data; + let (max_iters, typical) = (data.x().max(), data.y().max()); + let mut scaled_y: Vec<f64> = data.y().iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_y); + let scaled_y = Sample::new(&scaled_y); + + let point_estimate = Slope::fit(&measurements.data).0; + let mut scaled_points = [point_estimate * max_iters, lb * max_iters, ub * max_iters]; + let _ = formatter.scale_values(typical, &mut scaled_points); + let [point, lb, ub] = scaled_points; + + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let x_scale = 10f64.powi(-exponent); + + let x_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + a.configure(Grid::Major, |g| g.show()) + .set(Label(x_label)) + .set(ScaleFactor(x_scale)) + }) + .configure(Axis::LeftY, |a| { + a.configure(Grid::Major, |g| g.show()) + .set(Label(format!("Total sample time ({})", unit))) + }) + .plot( + Points { + x: data.x().as_ref(), + y: scaled_y.as_ref(), + }, + |c| { + c.set(DARK_BLUE) + .set(Label("Sample")) + .set(PointSize(0.5)) + .set(PointType::FilledCircle) + }, + ) + .plot( + Lines { + x: &[0., max_iters], + y: &[0., point], + }, + |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("Linear regression")) + .set(LineType::Solid) + }, + ) + .plot( + FilledCurve { + x: &[0., max_iters], + y1: &[0., lb], + y2: &[0., ub], + }, + |c| { + c.set(DARK_BLUE) + .set(Label("Confidence interval")) + .set(Opacity(0.25)) + }, + ); + figure +} + +pub(crate) fn regression( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Child { + let mut figure = regression_figure(formatter, measurements, size); + figure.set(Title(gnuplot_escape(id.as_title()))); + figure.configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Inside(Vertical::Top, Horizontal::Left)) + }); + + let path = context.report_path(id, "regression.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn regression_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<Size>, +) -> Child { + let mut figure = regression_figure(formatter, measurements, size); + figure.configure(Key, |k| k.hide()); + + let path = context.report_path(id, "regression_small.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +fn regression_comparison_figure( + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + base_data: &Data<'_, f64, f64>, + size: Option<Size>, +) -> Figure { + let data = &measurements.data; + let max_iters = base_data.x().max().max(data.x().max()); + let typical = base_data.y().max().max(data.y().max()); + + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let x_scale = 10f64.powi(-exponent); + + let x_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let Estimate { + confidence_interval: + ConfidenceInterval { + lower_bound: base_lb, + upper_bound: base_ub, + .. + }, + point_estimate: base_point, + .. + } = comparison.base_estimates.slope.as_ref().unwrap(); + + let Estimate { + confidence_interval: + ConfidenceInterval { + lower_bound: lb, + upper_bound: ub, + .. + }, + point_estimate: point, + .. + } = measurements.absolute_estimates.slope.as_ref().unwrap(); + + let mut points = [ + base_lb * max_iters, + base_point * max_iters, + base_ub * max_iters, + lb * max_iters, + point * max_iters, + ub * max_iters, + ]; + let unit = formatter.scale_values(typical, &mut points); + let [base_lb, base_point, base_ub, lb, point, ub] = points; + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .configure(Axis::BottomX, |a| { + a.configure(Grid::Major, |g| g.show()) + .set(Label(x_label)) + .set(ScaleFactor(x_scale)) + }) + .configure(Axis::LeftY, |a| { + a.configure(Grid::Major, |g| g.show()) + .set(Label(format!("Total sample time ({})", unit))) + }) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Inside(Vertical::Top, Horizontal::Left)) + }) + .plot( + FilledCurve { + x: &[0., max_iters], + y1: &[0., base_lb], + y2: &[0., base_ub], + }, + |c| c.set(DARK_RED).set(Opacity(0.25)), + ) + .plot( + FilledCurve { + x: &[0., max_iters], + y1: &[0., lb], + y2: &[0., ub], + }, + |c| c.set(DARK_BLUE).set(Opacity(0.25)), + ) + .plot( + Lines { + x: &[0., max_iters], + y: &[0., base_point], + }, + |c| { + c.set(DARK_RED) + .set(LINEWIDTH) + .set(Label("Base sample")) + .set(LineType::Solid) + }, + ) + .plot( + Lines { + x: &[0., max_iters], + y: &[0., point], + }, + |c| { + c.set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("New sample")) + .set(LineType::Solid) + }, + ); + figure +} + +pub(crate) fn regression_comparison( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + base_data: &Data<'_, f64, f64>, + size: Option<Size>, +) -> Child { + let mut figure = + regression_comparison_figure(formatter, measurements, comparison, base_data, size); + figure.set(Title(gnuplot_escape(id.as_title()))); + + let path = context.report_path(id, "both/regression.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} + +pub(crate) fn regression_comparison_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + base_data: &Data<'_, f64, f64>, + size: Option<Size>, +) -> Child { + let mut figure = + regression_comparison_figure(formatter, measurements, comparison, base_data, size); + figure.configure(Key, |k| k.hide()); + + let path = context.report_path(id, "relative_regression_small.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} diff --git a/src/plot/gnuplot_backend/summary.rs b/src/plot/gnuplot_backend/summary.rs new file mode 100755 index 0000000..cdd0604 --- /dev/null +++ b/src/plot/gnuplot_backend/summary.rs @@ -0,0 +1,211 @@ +use super::{debug_script, gnuplot_escape}; +use super::{DARK_BLUE, DEFAULT_FONT, KDE_POINTS, LINEWIDTH, POINT_SIZE, SIZE}; +use crate::kde; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ValueType}; +use crate::stats::univariate::Sample; +use crate::AxisScale; +use criterion_plot::prelude::*; +use itertools::Itertools; +use std::cmp::Ordering; +use std::path::{Path, PathBuf}; +use std::process::Child; + +const NUM_COLORS: usize = 8; +static COMPARISON_COLORS: [Color; NUM_COLORS] = [ + Color::Rgb(178, 34, 34), + Color::Rgb(46, 139, 87), + Color::Rgb(0, 139, 139), + Color::Rgb(255, 215, 0), + Color::Rgb(0, 0, 139), + Color::Rgb(220, 20, 60), + Color::Rgb(139, 0, 139), + Color::Rgb(0, 255, 127), +]; + +impl AxisScale { + fn to_gnuplot(self) -> Scale { + match self { + AxisScale::Linear => Scale::Linear, + AxisScale::Logarithmic => Scale::Logarithmic, + } + } +} + +#[cfg_attr(feature = "cargo-clippy", allow(clippy::explicit_counter_loop))] +pub fn line_comparison( + formatter: &dyn ValueFormatter, + title: &str, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + path: &Path, + value_type: ValueType, + axis_scale: AxisScale, +) -> Child { + let path = PathBuf::from(path); + let mut f = Figure::new(); + + let input_suffix = match value_type { + ValueType::Bytes => " Size (Bytes)", + ValueType::Elements => " Size (Elements)", + ValueType::Value => "", + }; + + f.set(Font(DEFAULT_FONT)) + .set(SIZE) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Outside(Vertical::Top, Horizontal::Right)) + }) + .set(Title(format!("{}: Comparison", gnuplot_escape(title)))) + .configure(Axis::BottomX, |a| { + a.set(Label(format!("Input{}", input_suffix))) + .set(axis_scale.to_gnuplot()) + }); + + let mut i = 0; + + let max = all_curves + .iter() + .map(|&&(_, ref data)| Sample::new(data).mean()) + .fold(::std::f64::NAN, f64::max); + + let mut dummy = [1.0]; + let unit = formatter.scale_values(max, &mut dummy); + + f.configure(Axis::LeftY, |a| { + a.configure(Grid::Major, |g| g.show()) + .configure(Grid::Minor, |g| g.hide()) + .set(Label(format!("Average time ({})", unit))) + .set(axis_scale.to_gnuplot()) + }); + + // This assumes the curves are sorted. It also assumes that the benchmark IDs all have numeric + // values or throughputs and that value is sensible (ie. not a mix of bytes and elements + // or whatnot) + for (key, group) in &all_curves.iter().group_by(|&&&(ref id, _)| &id.function_id) { + let mut tuples: Vec<_> = group + .map(|&&(ref id, ref sample)| { + // Unwrap is fine here because it will only fail if the assumptions above are not true + // ie. programmer error. + let x = id.as_number().unwrap(); + let y = Sample::new(sample).mean(); + + (x, y) + }) + .collect(); + tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less))); + let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); + formatter.scale_values(max, &mut ys); + + let function_name = key.as_ref().map(|string| gnuplot_escape(string)); + + f.plot(Lines { x: &xs, y: &ys }, |c| { + if let Some(name) = function_name { + c.set(Label(name)); + } + c.set(LINEWIDTH) + .set(LineType::Solid) + .set(COMPARISON_COLORS[i % NUM_COLORS]) + }) + .plot(Points { x: &xs, y: &ys }, |p| { + p.set(PointType::FilledCircle) + .set(POINT_SIZE) + .set(COMPARISON_COLORS[i % NUM_COLORS]) + }); + + i += 1; + } + + debug_script(&path, &f); + f.set(Output(path)).draw().unwrap() +} + +pub fn violin( + formatter: &dyn ValueFormatter, + title: &str, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + path: &Path, + axis_scale: AxisScale, +) -> Child { + let path = PathBuf::from(&path); + let all_curves_vec = all_curves.iter().rev().cloned().collect::<Vec<_>>(); + let all_curves: &[&(&BenchmarkId, Vec<f64>)] = &*all_curves_vec; + + let kdes = all_curves + .iter() + .map(|&&(_, ref sample)| { + let (x, mut y) = kde::sweep(Sample::new(sample), KDE_POINTS, None); + let y_max = Sample::new(&y).max(); + for y in y.iter_mut() { + *y /= y_max; + } + + (x, y) + }) + .collect::<Vec<_>>(); + let mut xs = kdes + .iter() + .flat_map(|&(ref x, _)| x.iter()) + .filter(|&&x| x > 0.); + let (mut min, mut max) = { + let &first = xs.next().unwrap(); + (first, first) + }; + for &e in xs { + if e < min { + min = e; + } else if e > max { + max = e; + } + } + let mut one = [1.0]; + // Scale the X axis units. Use the middle as a "typical value". E.g. if + // it is 0.002 s then this function will decide that milliseconds are an + // appropriate unit. It will multiple `one` by 1000, and return "ms". + let unit = formatter.scale_values((min + max) / 2.0, &mut one); + + let tics = || (0..).map(|x| (f64::from(x)) + 0.5); + let size = Size(1280, 200 + (25 * all_curves.len())); + let mut f = Figure::new(); + f.set(Font(DEFAULT_FONT)) + .set(size) + .set(Title(format!("{}: Violin plot", gnuplot_escape(title)))) + .configure(Axis::BottomX, |a| { + a.configure(Grid::Major, |g| g.show()) + .configure(Grid::Minor, |g| g.hide()) + .set(Label(format!("Average time ({})", unit))) + .set(axis_scale.to_gnuplot()) + }) + .configure(Axis::LeftY, |a| { + a.set(Label("Input")) + .set(Range::Limits(0., all_curves.len() as f64)) + .set(TicLabels { + positions: tics(), + labels: all_curves + .iter() + .map(|&&(ref id, _)| gnuplot_escape(id.as_title())), + }) + }); + + let mut is_first = true; + for (i, &(ref x, ref y)) in kdes.iter().enumerate() { + let i = i as f64 + 0.5; + let y1: Vec<_> = y.iter().map(|&y| i + y * 0.45).collect(); + let y2: Vec<_> = y.iter().map(|&y| i - y * 0.45).collect(); + + let x: Vec<_> = x.iter().map(|&x| x * one[0]).collect(); + + f.plot(FilledCurve { x, y1, y2 }, |c| { + if is_first { + is_first = false; + + c.set(DARK_BLUE).set(Label("PDF")).set(Opacity(0.25)) + } else { + c.set(DARK_BLUE).set(Opacity(0.25)) + } + }); + } + debug_script(&path, &f); + f.set(Output(path)).draw().unwrap() +} diff --git a/src/plot/gnuplot_backend/t_test.rs b/src/plot/gnuplot_backend/t_test.rs new file mode 100755 index 0000000..47b4a11 --- /dev/null +++ b/src/plot/gnuplot_backend/t_test.rs @@ -0,0 +1,65 @@ +use std::iter; +use std::process::Child; + +use criterion_plot::prelude::*; + +use super::*; +use crate::kde; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; + +pub(crate) fn t_test( + id: &BenchmarkId, + context: &ReportContext, + _measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<Size>, +) -> Child { + let t = comparison.t_value; + let (xs, ys) = kde::sweep(&comparison.t_distribution, KDE_POINTS, None); + let zero = iter::repeat(0); + + let mut figure = Figure::new(); + figure + .set(Font(DEFAULT_FONT)) + .set(size.unwrap_or(SIZE)) + .set(Title(format!( + "{}: Welch t test", + gnuplot_escape(id.as_title()) + ))) + .configure(Axis::BottomX, |a| a.set(Label("t score"))) + .configure(Axis::LeftY, |a| a.set(Label("Density"))) + .configure(Key, |k| { + k.set(Justification::Left) + .set(Order::SampleText) + .set(Position::Outside(Vertical::Top, Horizontal::Right)) + }) + .plot( + FilledCurve { + x: &*xs, + y1: &*ys, + y2: zero, + }, + |c| { + c.set(DARK_BLUE) + .set(Label("t distribution")) + .set(Opacity(0.25)) + }, + ) + .plot( + Lines { + x: &[t, t], + y: &[0, 1], + }, + |c| { + c.set(Axes::BottomXRightY) + .set(DARK_BLUE) + .set(LINEWIDTH) + .set(Label("t statistic")) + .set(LineType::Solid) + }, + ); + + let path = context.report_path(id, "change/t-test.svg"); + debug_script(&path, &figure); + figure.set(Output(path)).draw().unwrap() +} diff --git a/src/plot/mod.rs b/src/plot/mod.rs new file mode 100755 index 0000000..cb836a3 --- /dev/null +++ b/src/plot/mod.rs @@ -0,0 +1,103 @@ +mod gnuplot_backend; +mod plotters_backend; + +pub(crate) use gnuplot_backend::Gnuplot; +pub(crate) use plotters_backend::PlottersBackend; + +use crate::estimate::Statistic; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext, ValueType}; +use std::path::PathBuf; + +const REPORT_STATS: [Statistic; 7] = [ + Statistic::Typical, + Statistic::Slope, + Statistic::Mean, + Statistic::Median, + Statistic::MedianAbsDev, + Statistic::MedianAbsDev, + Statistic::StdDev, +]; +const CHANGE_STATS: [Statistic; 2] = [Statistic::Mean, Statistic::Median]; +#[derive(Clone, Copy)] +pub(crate) struct PlotContext<'a> { + pub(crate) id: &'a BenchmarkId, + pub(crate) context: &'a ReportContext, + pub(crate) size: Option<(usize, usize)>, + pub(crate) is_thumbnail: bool, +} + +impl<'a> PlotContext<'a> { + pub fn size(mut self, s: Option<criterion_plot::Size>) -> PlotContext<'a> { + if let Some(s) = s { + self.size = Some((s.0, s.1)); + } + self + } + + pub fn thumbnail(mut self, value: bool) -> PlotContext<'a> { + self.is_thumbnail = value; + self + } + + pub fn line_comparison_path(&self) -> PathBuf { + let mut path = self.context.output_directory.clone(); + path.push(self.id.as_directory_name()); + path.push("report"); + path.push("lines.svg"); + path + } + + pub fn violin_path(&self) -> PathBuf { + let mut path = self.context.output_directory.clone(); + path.push(self.id.as_directory_name()); + path.push("report"); + path.push("violin.svg"); + path + } +} + +#[derive(Clone, Copy)] +pub(crate) struct PlotData<'a> { + pub(crate) formatter: &'a dyn ValueFormatter, + pub(crate) measurements: &'a MeasurementData<'a>, + pub(crate) comparison: Option<&'a ComparisonData>, +} + +impl<'a> PlotData<'a> { + pub fn comparison(mut self, comp: &'a ComparisonData) -> PlotData<'a> { + self.comparison = Some(comp); + self + } +} + +pub(crate) trait Plotter { + fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn iteration_times(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn line_comparison( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + value_type: ValueType, + ); + + fn violin( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + ); + + fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn wait(&mut self); +} diff --git a/src/plot/plotters_backend/distributions.rs b/src/plot/plotters_backend/distributions.rs new file mode 100755 index 0000000..ed4d61a --- /dev/null +++ b/src/plot/plotters_backend/distributions.rs @@ -0,0 +1,309 @@ +use super::*; +use crate::estimate::Estimate; +use crate::estimate::Statistic; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, MeasurementData, ReportContext}; +use crate::stats::Distribution; + +fn abs_distribution( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + statistic: Statistic, + distribution: &Distribution<f64>, + estimate: &Estimate, + size: Option<(u32, u32)>, +) { + let ci = &estimate.confidence_interval; + let typical = ci.upper_bound; + let mut ci_values = [ci.lower_bound, ci.upper_bound, estimate.point_estimate]; + let unit = formatter.scale_values(typical, &mut ci_values); + let (lb, ub, point) = (ci_values[0], ci_values[1], ci_values[2]); + + let start = lb - (ub - lb) / 9.; + let end = ub + (ub - lb) / 9.; + let mut scaled_xs: Vec<f64> = distribution.iter().cloned().collect(); + let _ = formatter.scale_values(typical, &mut scaled_xs); + let scaled_xs_sample = Sample::new(&scaled_xs); + let (kde_xs, ys) = kde::sweep(scaled_xs_sample, KDE_POINTS, Some((start, end))); + + // interpolate between two points of the KDE sweep to find the Y position at the point estimate. + let n_point = kde_xs + .iter() + .position(|&x| x >= point) + .unwrap_or(kde_xs.len() - 1) + .max(1); // Must be at least the second element or this will panic + let slope = (ys[n_point] - ys[n_point - 1]) / (kde_xs[n_point] - kde_xs[n_point - 1]); + let y_point = ys[n_point - 1] + (slope * (point - kde_xs[n_point - 1])); + + let start = kde_xs + .iter() + .enumerate() + .find(|&(_, &x)| x >= lb) + .unwrap() + .0; + let end = kde_xs + .iter() + .enumerate() + .rev() + .find(|&(_, &x)| x <= ub) + .unwrap() + .0; + let len = end - start; + + let kde_xs_sample = Sample::new(&kde_xs); + + let path = context.report_path(id, &format!("{}.svg", statistic)); + let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area(); + + let x_range = plotters::data::fitting_range(kde_xs_sample.iter()); + let mut y_range = plotters::data::fitting_range(ys.iter()); + + y_range.end *= 1.1; + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption( + format!("{}:{}", id.as_title(), statistic), + (DEFAULT_FONT, 20), + ) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .x_desc(format!("Average time ({})", unit)) + .y_desc("Density (a.u.)") + .x_label_formatter(&|&v| pretty_print_float(v, true)) + .y_label_formatter(&|&v| pretty_print_float(v, true)) + .draw() + .unwrap(); + + chart + .draw_series(LineSeries::new( + kde_xs.iter().zip(ys.iter()).map(|(&x, &y)| (x, y)), + &DARK_BLUE, + )) + .unwrap() + .label("Bootstrap distribution") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(AreaSeries::new( + kde_xs + .iter() + .zip(ys.iter()) + .skip(start) + .take(len) + .map(|(&x, &y)| (x, y)), + 0.0, + DARK_BLUE.mix(0.25).filled().stroke_width(3), + )) + .unwrap() + .label("Confidence interval") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(point, 0.0), (point, y_point)], + DARK_BLUE.filled().stroke_width(3), + ))) + .unwrap() + .label("Point estimate") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperRight) + .draw() + .unwrap(); +} + +pub(crate) fn abs_distributions( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + crate::plot::REPORT_STATS + .iter() + .filter_map(|stat| { + measurements.distributions.get(*stat).and_then(|dist| { + measurements + .absolute_estimates + .get(*stat) + .map(|est| (*stat, dist, est)) + }) + }) + .for_each(|(statistic, distribution, estimate)| { + abs_distribution( + id, + context, + formatter, + statistic, + distribution, + estimate, + size, + ) + }) +} + +fn rel_distribution( + id: &BenchmarkId, + context: &ReportContext, + statistic: Statistic, + distribution: &Distribution<f64>, + estimate: &Estimate, + noise_threshold: f64, + size: Option<(u32, u32)>, +) { + let ci = &estimate.confidence_interval; + let (lb, ub) = (ci.lower_bound, ci.upper_bound); + + let start = lb - (ub - lb) / 9.; + let end = ub + (ub - lb) / 9.; + let (xs, ys) = kde::sweep(distribution, KDE_POINTS, Some((start, end))); + let xs_ = Sample::new(&xs); + + // interpolate between two points of the KDE sweep to find the Y position at the point estimate. + let point = estimate.point_estimate; + let n_point = xs + .iter() + .position(|&x| x >= point) + .unwrap_or(ys.len() - 1) + .max(1); + let slope = (ys[n_point] - ys[n_point - 1]) / (xs[n_point] - xs[n_point - 1]); + let y_point = ys[n_point - 1] + (slope * (point - xs[n_point - 1])); + + let start = xs.iter().enumerate().find(|&(_, &x)| x >= lb).unwrap().0; + let end = xs + .iter() + .enumerate() + .rev() + .find(|&(_, &x)| x <= ub) + .unwrap() + .0; + let len = end - start; + + let x_min = xs_.min(); + let x_max = xs_.max(); + + let (fc_start, fc_end) = if noise_threshold < x_min || -noise_threshold > x_max { + let middle = (x_min + x_max) / 2.; + + (middle, middle) + } else { + ( + if -noise_threshold < x_min { + x_min + } else { + -noise_threshold + }, + if noise_threshold > x_max { + x_max + } else { + noise_threshold + }, + ) + }; + let y_range = plotters::data::fitting_range(ys.iter()); + let path = context.report_path(id, &format!("change/{}.svg", statistic)); + let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area(); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption( + format!("{}:{}", id.as_title(), statistic), + (DEFAULT_FONT, 20), + ) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_min..x_max, y_range.clone()) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .x_desc("Relative change (%)") + .y_desc("Density (a.u.)") + .x_label_formatter(&|&v| pretty_print_float(v, true)) + .y_label_formatter(&|&v| pretty_print_float(v, true)) + .draw() + .unwrap(); + + chart + .draw_series(LineSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + &DARK_BLUE, + )) + .unwrap() + .label("Bootstrap distribution") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(AreaSeries::new( + xs.iter() + .zip(ys.iter()) + .skip(start) + .take(len) + .map(|(x, y)| (*x, *y)), + 0.0, + DARK_BLUE.mix(0.25).filled().stroke_width(3), + )) + .unwrap() + .label("Confidence interval") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(point, 0.0), (point, y_point)], + DARK_BLUE.filled().stroke_width(3), + ))) + .unwrap() + .label("Point estimate") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(std::iter::once(Rectangle::new( + [(fc_start, y_range.start), (fc_end, y_range.end)], + DARK_RED.mix(0.1).filled(), + ))) + .unwrap() + .label("Noise threshold") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_RED.mix(0.25).filled()) + }); + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperRight) + .draw() + .unwrap(); +} + +pub(crate) fn rel_distributions( + id: &BenchmarkId, + context: &ReportContext, + _measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + crate::plot::CHANGE_STATS.iter().for_each(|&statistic| { + rel_distribution( + id, + context, + statistic, + comparison.relative_distributions.get(statistic), + comparison.relative_estimates.get(statistic), + comparison.noise_threshold, + size, + ) + }); +} diff --git a/src/plot/plotters_backend/iteration_times.rs b/src/plot/plotters_backend/iteration_times.rs new file mode 100755 index 0000000..a2204df --- /dev/null +++ b/src/plot/plotters_backend/iteration_times.rs @@ -0,0 +1,138 @@ +use super::*; + +use std::path::Path; + +pub(crate) fn iteration_times_figure( + title: Option<&str>, + path: &Path, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let data = &measurements.avg_times; + let max_avg_time = data.max(); + let mut scaled_y: Vec<_> = data.iter().map(|(f, _)| f).collect(); + let unit = formatter.scale_values(max_avg_time, &mut scaled_y); + let scaled_y = Sample::new(&scaled_y); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(path, size).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let x_range = (1.0)..((data.len() + 1) as f64); + let y_range = plotters::data::fitting_range(scaled_y.iter()); + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .y_desc(format!("Average Iteration Time ({})", unit)) + .x_label_formatter(&|x| pretty_print_float(*x, true)) + .line_style_2(&TRANSPARENT) + .draw() + .unwrap(); + + chart + .draw_series( + (1..=data.len()) + .zip(scaled_y.iter()) + .map(|(x, y)| Circle::new((x as f64, *y), POINT_SIZE, DARK_BLUE.filled())), + ) + .unwrap() + .label("Sample") + .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled())); + + if title.is_some() { + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); + } +} + +pub(crate) fn iteration_times_comparison_figure( + title: Option<&str>, + path: &Path, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + let current_data = &measurements.avg_times; + let base_data = &comparison.base_avg_times; + + let mut all_data: Vec<f64> = current_data.iter().map(|(f, _)| f).collect(); + all_data.extend_from_slice(base_data); + + let typical_value = Sample::new(&all_data).max(); + let unit = formatter.scale_values(typical_value, &mut all_data); + + let (scaled_current_y, scaled_base_y) = all_data.split_at(current_data.len()); + let scaled_current_y = Sample::new(scaled_current_y); + let scaled_base_y = Sample::new(scaled_base_y); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(path, size).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let max_samples = current_data.len().max(base_data.len()) as f64; + + let y_range = plotters::data::fitting_range(all_data.iter()); + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(0.0..max_samples, y_range) + .unwrap(); + + chart + .configure_mesh() + .y_desc(format!("Average Iteration Time ({})", unit)) + .x_label_formatter(&|x| pretty_print_float(*x, true)) + .line_style_2(&TRANSPARENT) + .draw() + .unwrap(); + + chart + .draw_series( + (1..=current_data.len()) + .zip(scaled_current_y.iter()) + .map(|(x, y)| Circle::new((x as f64, *y), POINT_SIZE, DARK_BLUE.filled())), + ) + .unwrap() + .label("Current") + .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled())); + + chart + .draw_series( + (1..=base_data.len()) + .zip(scaled_base_y.iter()) + .map(|(x, y)| Circle::new((x as f64, *y), POINT_SIZE, DARK_RED.filled())), + ) + .unwrap() + .label("Base") + .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_RED.filled())); + + if title.is_some() { + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); + } +} diff --git a/src/plot/plotters_backend/mod.rs b/src/plot/plotters_backend/mod.rs new file mode 100755 index 0000000..4cd1b18 --- /dev/null +++ b/src/plot/plotters_backend/mod.rs @@ -0,0 +1,232 @@ +use super::{PlotContext, PlotData, Plotter}; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ValueType}; +use plotters::data::float::pretty_print_float; +use plotters::prelude::*; + +use crate::kde; +use crate::stats::bivariate::Data; +use crate::stats::univariate::Sample; + +static DEFAULT_FONT: FontFamily = FontFamily::SansSerif; +static KDE_POINTS: usize = 500; +static SIZE: (u32, u32) = (960, 540); +static POINT_SIZE: u32 = 3; + +const DARK_BLUE: RGBColor = RGBColor(31, 120, 180); +const DARK_ORANGE: RGBColor = RGBColor(255, 127, 0); +const DARK_RED: RGBColor = RGBColor(227, 26, 28); + +mod distributions; +mod iteration_times; +mod pdf; +mod regression; +mod summary; +mod t_test; + +fn convert_size(size: Option<(usize, usize)>) -> Option<(u32, u32)> { + if let Some((w, h)) = size { + return Some((w as u32, h as u32)); + } + None +} +#[derive(Default)] +pub struct PlottersBackend; + +#[allow(unused_variables)] +impl Plotter for PlottersBackend { + fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + if let Some(cmp) = data.comparison { + let (path, title) = if ctx.is_thumbnail { + ( + ctx.context.report_path(ctx.id, "relative_pdf_small.svg"), + None, + ) + } else { + ( + ctx.context.report_path(ctx.id, "both/pdf.svg"), + Some(ctx.id.as_title()), + ) + }; + pdf::pdf_comparison_figure( + path.as_ref(), + title, + data.formatter, + data.measurements, + cmp, + convert_size(ctx.size), + ); + return; + } + if ctx.is_thumbnail { + pdf::pdf_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } else { + pdf::pdf( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + } + + fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let (title, path) = match (data.comparison.is_some(), ctx.is_thumbnail) { + (true, true) => ( + None, + ctx.context + .report_path(ctx.id, "relative_regression_small.svg"), + ), + (true, false) => ( + Some(ctx.id.as_title()), + ctx.context.report_path(ctx.id, "both/regression.svg"), + ), + (false, true) => ( + None, + ctx.context.report_path(ctx.id, "regression_small.svg"), + ), + (false, false) => ( + Some(ctx.id.as_title()), + ctx.context.report_path(ctx.id, "regression.svg"), + ), + }; + + if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + regression::regression_comparison_figure( + title, + path.as_path(), + data.formatter, + data.measurements, + cmp, + &base_data, + convert_size(ctx.size), + ); + } else { + regression::regression_figure( + title, + path.as_path(), + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + } + + fn iteration_times(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let (title, path) = match (data.comparison.is_some(), ctx.is_thumbnail) { + (true, true) => ( + None, + ctx.context + .report_path(ctx.id, "relative_iteration_times_small.svg"), + ), + (true, false) => ( + Some(ctx.id.as_title()), + ctx.context.report_path(ctx.id, "both/iteration_times.svg"), + ), + (false, true) => ( + None, + ctx.context.report_path(ctx.id, "iteration_times_small.svg"), + ), + (false, false) => ( + Some(ctx.id.as_title()), + ctx.context.report_path(ctx.id, "iteration_times.svg"), + ), + }; + + if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + iteration_times::iteration_times_comparison_figure( + title, + path.as_path(), + data.formatter, + data.measurements, + cmp, + convert_size(ctx.size), + ); + } else { + iteration_times::iteration_times_figure( + title, + path.as_path(), + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + } + + fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + distributions::abs_distributions( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + + fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + distributions::rel_distributions( + ctx.id, + ctx.context, + data.measurements, + data.comparison.unwrap(), + convert_size(ctx.size), + ); + } + + fn line_comparison( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + value_type: ValueType, + ) { + let path = ctx.line_comparison_path(); + summary::line_comparison( + formatter, + ctx.id.as_title(), + all_curves, + &path, + value_type, + ctx.context.plot_config.summary_scale, + ); + } + + fn violin( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + ) { + let violin_path = ctx.violin_path(); + + summary::violin( + formatter, + ctx.id.as_title(), + all_curves, + &violin_path, + ctx.context.plot_config.summary_scale, + ); + } + + fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let title = ctx.id.as_title(); + let path = ctx.context.report_path(ctx.id, "change/t-test.svg"); + t_test::t_test( + path.as_path(), + title, + data.comparison.unwrap(), + convert_size(ctx.size), + ); + } + + fn wait(&mut self) {} +} diff --git a/src/plot/plotters_backend/pdf.rs b/src/plot/plotters_backend/pdf.rs new file mode 100755 index 0000000..d8f3541 --- /dev/null +++ b/src/plot/plotters_backend/pdf.rs @@ -0,0 +1,307 @@ +use super::*; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; +use plotters::data; +use plotters::style::RGBAColor; +use std::path::Path; + +pub(crate) fn pdf_comparison_figure( + path: &Path, + title: Option<&str>, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + let base_avg_times = Sample::new(&comparison.base_avg_times); + let typical = base_avg_times.max().max(measurements.avg_times.max()); + let mut scaled_base_avg_times: Vec<f64> = comparison.base_avg_times.clone(); + let unit = formatter.scale_values(typical, &mut scaled_base_avg_times); + let scaled_base_avg_times = Sample::new(&scaled_base_avg_times); + + let mut scaled_new_avg_times: Vec<f64> = (&measurements.avg_times as &Sample<f64>) + .iter() + .cloned() + .collect(); + let _ = formatter.scale_values(typical, &mut scaled_new_avg_times); + let scaled_new_avg_times = Sample::new(&scaled_new_avg_times); + + let base_mean = scaled_base_avg_times.mean(); + let new_mean = scaled_new_avg_times.mean(); + + let (base_xs, base_ys, base_y_mean) = + kde::sweep_and_estimate(scaled_base_avg_times, KDE_POINTS, None, base_mean); + let (xs, ys, y_mean) = + kde::sweep_and_estimate(scaled_new_avg_times, KDE_POINTS, None, new_mean); + + let x_range = data::fitting_range(base_xs.iter().chain(xs.iter())); + let y_range = data::fitting_range(base_ys.iter().chain(ys.iter())); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range.clone()) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Density (a.u.)") + .x_desc(format!("Average Time ({})", unit)) + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y, true)) + .x_labels(5) + .draw() + .unwrap(); + + chart + .draw_series(AreaSeries::new( + base_xs.iter().zip(base_ys.iter()).map(|(x, y)| (*x, *y)), + y_range.start, + DARK_RED.mix(0.5).filled(), + )) + .unwrap() + .label("Base PDF") + .legend(|(x, y)| Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_RED.mix(0.5).filled())); + + chart + .draw_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + y_range.start, + DARK_BLUE.mix(0.5).filled(), + )) + .unwrap() + .label("New PDF") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.5).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(base_mean, 0.0), (base_mean, base_y_mean)], + DARK_RED.filled().stroke_width(2), + ))) + .unwrap() + .label("Base Mean") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_RED)); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(new_mean, 0.0), (new_mean, y_mean)], + DARK_BLUE.filled().stroke_width(2), + ))) + .unwrap() + .label("New Mean") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + if title.is_some() { + chart.configure_series_labels().draw().unwrap(); + } +} + +pub(crate) fn pdf_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let avg_times = &*measurements.avg_times; + let typical = avg_times.max(); + let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_avg_times); + let scaled_avg_times = Sample::new(&scaled_avg_times); + let mean = scaled_avg_times.mean(); + + let (xs, ys, mean_y) = kde::sweep_and_estimate(scaled_avg_times, KDE_POINTS, None, mean); + let xs_ = Sample::new(&xs); + let ys_ = Sample::new(&ys); + + let y_limit = ys_.max() * 1.1; + + let path = context.report_path(id, "pdf_small.svg"); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area(); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(xs_.min()..xs_.max(), 0.0..y_limit) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Density (a.u.)") + .x_desc(format!("Average Time ({})", unit)) + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y, true)) + .x_labels(5) + .draw() + .unwrap(); + + chart + .draw_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + 0.0, + DARK_BLUE.mix(0.25).filled(), + )) + .unwrap(); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(mean, 0.0), (mean, mean_y)], + DARK_BLUE.filled().stroke_width(2), + ))) + .unwrap(); +} + +pub(crate) fn pdf( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let avg_times = &measurements.avg_times; + let typical = avg_times.max(); + let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_avg_times); + let scaled_avg_times = Sample::new(&scaled_avg_times); + + let mean = scaled_avg_times.mean(); + + let iter_counts = measurements.iter_counts(); + let &max_iters = iter_counts + .iter() + .max_by_key(|&&iters| iters as u64) + .unwrap(); + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let y_scale = 10f64.powi(-exponent); + + let y_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let (xs, ys) = kde::sweep(&scaled_avg_times, KDE_POINTS, None); + let (lost, lomt, himt, hist) = avg_times.fences(); + let mut fences = [lost, lomt, himt, hist]; + let _ = formatter.scale_values(typical, &mut fences); + let [lost, lomt, himt, hist] = fences; + + let path = context.report_path(id, "pdf.svg"); + + let xs_ = Sample::new(&xs); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area(); + + let range = data::fitting_range(ys.iter()); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption(id.as_title(), (DEFAULT_FONT, 20)) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Right, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(xs_.min()..xs_.max(), 0.0..max_iters) + .unwrap() + .set_secondary_coord(xs_.min()..xs_.max(), 0.0..range.end); + + chart + .configure_mesh() + .disable_mesh() + .y_desc(y_label) + .x_desc(format!("Average Time ({})", unit)) + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y * y_scale, true)) + .draw() + .unwrap(); + + chart + .configure_secondary_axes() + .y_desc("Density (a.u.)") + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y, true)) + .draw() + .unwrap(); + + chart + .draw_secondary_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + 0.0, + DARK_BLUE.mix(0.5).filled(), + )) + .unwrap() + .label("PDF") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.5).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(mean, 0.0), (mean, max_iters)], + &DARK_BLUE, + ))) + .unwrap() + .label("Mean") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(vec![ + PathElement::new(vec![(lomt, 0.0), (lomt, max_iters)], &DARK_ORANGE), + PathElement::new(vec![(himt, 0.0), (himt, max_iters)], &DARK_ORANGE), + PathElement::new(vec![(lost, 0.0), (lost, max_iters)], &DARK_RED), + PathElement::new(vec![(hist, 0.0), (hist, max_iters)], &DARK_RED), + ]) + .unwrap(); + use crate::stats::univariate::outliers::tukey::Label; + + let mut draw_data_point_series = + |filter: &dyn Fn(&Label) -> bool, color: RGBAColor, name: &str| { + chart + .draw_series( + avg_times + .iter() + .zip(scaled_avg_times.iter()) + .zip(iter_counts.iter()) + .filter_map(|(((_, label), t), i)| { + if filter(&label) { + Some(Circle::new((*t, *i), POINT_SIZE, color.filled())) + } else { + None + } + }), + ) + .unwrap() + .label(name) + .legend(move |(x, y)| Circle::new((x + 10, y), POINT_SIZE, color.filled())); + }; + + draw_data_point_series( + &|l| !l.is_outlier(), + DARK_BLUE.to_rgba(), + "\"Clean\" sample", + ); + draw_data_point_series( + &|l| l.is_mild(), + RGBColor(255, 127, 0).to_rgba(), + "Mild outliers", + ); + draw_data_point_series(&|l| l.is_severe(), DARK_RED.to_rgba(), "Severe outliers"); + chart.configure_series_labels().draw().unwrap(); +} diff --git a/src/plot/plotters_backend/regression.rs b/src/plot/plotters_backend/regression.rs new file mode 100755 index 0000000..54bffa6 --- /dev/null +++ b/src/plot/plotters_backend/regression.rs @@ -0,0 +1,234 @@ +use super::*; + +use std::path::Path; + +use crate::estimate::{ConfidenceInterval, Estimate}; +use crate::stats::bivariate::regression::Slope; +use crate::stats::bivariate::Data; + +pub(crate) fn regression_figure( + title: Option<&str>, + path: &Path, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let slope_estimate = measurements.absolute_estimates.slope.as_ref().unwrap(); + let slope_dist = measurements.distributions.slope.as_ref().unwrap(); + let (lb, ub) = + slope_dist.confidence_interval(slope_estimate.confidence_interval.confidence_level); + + let data = &measurements.data; + let (max_iters, typical) = (data.x().max(), data.y().max()); + let mut scaled_y: Vec<f64> = data.y().iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_y); + let scaled_y = Sample::new(&scaled_y); + + let point_estimate = Slope::fit(&measurements.data).0; + let mut scaled_points = [point_estimate * max_iters, lb * max_iters, ub * max_iters]; + let _ = formatter.scale_values(typical, &mut scaled_points); + let [point, lb, ub] = scaled_points; + + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + + let x_scale = 10f64.powi(-exponent); + let x_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(path, size).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let x_range = plotters::data::fitting_range(data.x().iter()); + let y_range = plotters::data::fitting_range(scaled_y.iter()); + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .x_desc(x_label) + .y_desc(format!("Total sample time ({})", unit)) + .x_label_formatter(&|x| pretty_print_float(x * x_scale, true)) + .line_style_2(&TRANSPARENT) + .draw() + .unwrap(); + + chart + .draw_series( + data.x() + .iter() + .zip(scaled_y.iter()) + .map(|(x, y)| Circle::new((*x, *y), POINT_SIZE, DARK_BLUE.filled())), + ) + .unwrap() + .label("Sample") + .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled())); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(0.0, 0.0), (max_iters, point)], + &DARK_BLUE, + ))) + .unwrap() + .label("Linear regression") + .legend(|(x, y)| { + PathElement::new( + vec![(x, y), (x + 20, y)], + DARK_BLUE.filled().stroke_width(2), + ) + }); + + chart + .draw_series(std::iter::once(Polygon::new( + vec![(0.0, 0.0), (max_iters, lb), (max_iters, ub)], + DARK_BLUE.mix(0.25).filled(), + ))) + .unwrap() + .label("Confidence interval") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + if title.is_some() { + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); + } +} + +pub(crate) fn regression_comparison_figure( + title: Option<&str>, + path: &Path, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + base_data: &Data<'_, f64, f64>, + size: Option<(u32, u32)>, +) { + let data = &measurements.data; + let max_iters = base_data.x().max().max(data.x().max()); + let typical = base_data.y().max().max(data.y().max()); + + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let x_scale = 10f64.powi(-exponent); + + let x_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let Estimate { + confidence_interval: + ConfidenceInterval { + lower_bound: base_lb, + upper_bound: base_ub, + .. + }, + point_estimate: base_point, + .. + } = comparison.base_estimates.slope.as_ref().unwrap(); + + let Estimate { + confidence_interval: + ConfidenceInterval { + lower_bound: lb, + upper_bound: ub, + .. + }, + point_estimate: point, + .. + } = measurements.absolute_estimates.slope.as_ref().unwrap(); + + let mut points = [ + base_lb * max_iters, + base_point * max_iters, + base_ub * max_iters, + lb * max_iters, + point * max_iters, + ub * max_iters, + ]; + let unit = formatter.scale_values(typical, &mut points); + let [base_lb, base_point, base_ub, lb, point, ub] = points; + + let y_max = point.max(base_point); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(path, size).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(0.0..max_iters, 0.0..y_max) + .unwrap(); + + chart + .configure_mesh() + .x_desc(x_label) + .y_desc(format!("Total sample time ({})", unit)) + .x_label_formatter(&|x| pretty_print_float(x * x_scale, true)) + .line_style_2(&TRANSPARENT) + .draw() + .unwrap(); + + chart + .draw_series(vec![ + PathElement::new(vec![(0.0, 0.0), (max_iters, base_point)], &DARK_RED).into_dyn(), + Polygon::new( + vec![(0.0, 0.0), (max_iters, base_lb), (max_iters, base_ub)], + DARK_RED.mix(0.25).filled(), + ) + .into_dyn(), + ]) + .unwrap() + .label("Base Sample") + .legend(|(x, y)| { + PathElement::new(vec![(x, y), (x + 20, y)], DARK_RED.filled().stroke_width(2)) + }); + + chart + .draw_series(vec![ + PathElement::new(vec![(0.0, 0.0), (max_iters, point)], &DARK_BLUE).into_dyn(), + Polygon::new( + vec![(0.0, 0.0), (max_iters, lb), (max_iters, ub)], + DARK_BLUE.mix(0.25).filled(), + ) + .into_dyn(), + ]) + .unwrap() + .label("New Sample") + .legend(|(x, y)| { + PathElement::new( + vec![(x, y), (x + 20, y)], + DARK_BLUE.filled().stroke_width(2), + ) + }); + + if title.is_some() { + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); + } +} diff --git a/src/plot/plotters_backend/summary.rs b/src/plot/plotters_backend/summary.rs new file mode 100755 index 0000000..dd02d0c --- /dev/null +++ b/src/plot/plotters_backend/summary.rs @@ -0,0 +1,256 @@ +use super::*; +use crate::AxisScale; +use itertools::Itertools; +use plotters::coord::{AsRangedCoord, Shift}; +use std::cmp::Ordering; +use std::path::Path; + +const NUM_COLORS: usize = 8; +static COMPARISON_COLORS: [RGBColor; NUM_COLORS] = [ + RGBColor(178, 34, 34), + RGBColor(46, 139, 87), + RGBColor(0, 139, 139), + RGBColor(255, 215, 0), + RGBColor(0, 0, 139), + RGBColor(220, 20, 60), + RGBColor(139, 0, 139), + RGBColor(0, 255, 127), +]; + +pub fn line_comparison( + formatter: &dyn ValueFormatter, + title: &str, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + path: &Path, + value_type: ValueType, + axis_scale: AxisScale, +) { + let (unit, series_data) = line_comparision_series_data(formatter, all_curves); + + let x_range = + plotters::data::fitting_range(series_data.iter().map(|(_, xs, _)| xs.iter()).flatten()); + let y_range = + plotters::data::fitting_range(series_data.iter().map(|(_, _, ys)| ys.iter()).flatten()); + let root_area = SVGBackend::new(&path, SIZE) + .into_drawing_area() + .titled(&format!("{}: Comparision", title), (DEFAULT_FONT, 20)) + .unwrap(); + + match axis_scale { + AxisScale::Linear => { + draw_line_comarision_figure(root_area, unit, x_range, y_range, value_type, series_data) + } + AxisScale::Logarithmic => draw_line_comarision_figure( + root_area, + unit, + LogRange(x_range), + LogRange(y_range), + value_type, + series_data, + ), + } +} + +fn draw_line_comarision_figure<XR: AsRangedCoord<Value = f64>, YR: AsRangedCoord<Value = f64>>( + root_area: DrawingArea<SVGBackend, Shift>, + y_unit: &str, + x_range: XR, + y_range: YR, + value_type: ValueType, + data: Vec<(Option<&String>, Vec<f64>, Vec<f64>)>, +) { + let input_suffix = match value_type { + ValueType::Bytes => " Size (Bytes)", + ValueType::Elements => " Size (Elements)", + ValueType::Value => "", + }; + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .x_desc(format!("Input{}", input_suffix)) + .y_desc(format!("Average time ({})", y_unit)) + .draw() + .unwrap(); + + for (id, (name, xs, ys)) in (0..).zip(data.into_iter()) { + let series = chart + .draw_series( + LineSeries::new( + xs.into_iter().zip(ys.into_iter()), + COMPARISON_COLORS[id % NUM_COLORS].filled(), + ) + .point_size(POINT_SIZE), + ) + .unwrap(); + if let Some(name) = name { + let name: &str = &*name; + series.label(name).legend(move |(x, y)| { + Rectangle::new( + [(x, y - 5), (x + 20, y + 5)], + COMPARISON_COLORS[id % NUM_COLORS].filled(), + ) + }); + } + } + + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); +} + +#[allow(clippy::type_complexity)] +fn line_comparision_series_data<'a>( + formatter: &dyn ValueFormatter, + all_curves: &[&(&'a BenchmarkId, Vec<f64>)], +) -> (&'static str, Vec<(Option<&'a String>, Vec<f64>, Vec<f64>)>) { + let max = all_curves + .iter() + .map(|&&(_, ref data)| Sample::new(data).mean()) + .fold(::std::f64::NAN, f64::max); + + let mut dummy = [1.0]; + let unit = formatter.scale_values(max, &mut dummy); + + let mut series_data = vec![]; + + // This assumes the curves are sorted. It also assumes that the benchmark IDs all have numeric + // values or throughputs and that value is sensible (ie. not a mix of bytes and elements + // or whatnot) + for (key, group) in &all_curves.iter().group_by(|&&&(ref id, _)| &id.function_id) { + let mut tuples: Vec<_> = group + .map(|&&(ref id, ref sample)| { + // Unwrap is fine here because it will only fail if the assumptions above are not true + // ie. programmer error. + let x = id.as_number().unwrap(); + let y = Sample::new(sample).mean(); + + (x, y) + }) + .collect(); + tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less))); + let function_name = key.as_ref(); + let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); + formatter.scale_values(max, &mut ys); + series_data.push((function_name, xs, ys)); + } + (unit, series_data) +} + +pub fn violin( + formatter: &dyn ValueFormatter, + title: &str, + all_curves: &[&(&BenchmarkId, Vec<f64>)], + path: &Path, + axis_scale: AxisScale, +) { + let all_curves_vec = all_curves.iter().rev().cloned().collect::<Vec<_>>(); + let all_curves: &[&(&BenchmarkId, Vec<f64>)] = &*all_curves_vec; + + let mut kdes = all_curves + .iter() + .map(|&&(ref id, ref sample)| { + let (x, mut y) = kde::sweep(Sample::new(sample), KDE_POINTS, None); + let y_max = Sample::new(&y).max(); + for y in y.iter_mut() { + *y /= y_max; + } + + (id.as_title(), x, y) + }) + .collect::<Vec<_>>(); + + let mut xs = kdes + .iter() + .flat_map(|&(_, ref x, _)| x.iter()) + .filter(|&&x| x > 0.); + let (mut min, mut max) = { + let &first = xs.next().unwrap(); + (first, first) + }; + for &e in xs { + if e < min { + min = e; + } else if e > max { + max = e; + } + } + let mut dummy = [1.0]; + let unit = formatter.scale_values(max, &mut dummy); + kdes.iter_mut().for_each(|&mut (_, ref mut xs, _)| { + formatter.scale_values(max, xs); + }); + + let x_range = plotters::data::fitting_range(kdes.iter().map(|(_, xs, _)| xs.iter()).flatten()); + let y_range = -0.5..all_curves.len() as f64 - 0.5; + + let size = (960, 150 + (18 * all_curves.len() as u32)); + + let root_area = SVGBackend::new(&path, size) + .into_drawing_area() + .titled(&format!("{}: Violin plot", title), (DEFAULT_FONT, 20)) + .unwrap(); + + match axis_scale { + AxisScale::Linear => draw_violin_figure(root_area, unit, x_range, y_range, kdes), + AxisScale::Logarithmic => { + draw_violin_figure(root_area, unit, LogRange(x_range), y_range, kdes) + } + } +} + +#[allow(clippy::type_complexity)] +fn draw_violin_figure<XR: AsRangedCoord<Value = f64>, YR: AsRangedCoord<Value = f64>>( + root_area: DrawingArea<SVGBackend, Shift>, + unit: &'static str, + x_range: XR, + y_range: YR, + data: Vec<(&str, Box<[f64]>, Box<[f64]>)>, +) { + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (10).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_width().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Input") + .x_desc(format!("Average time ({})", unit)) + .y_label_style((DEFAULT_FONT, 10)) + .y_label_formatter(&|v: &f64| data[v.round() as usize].0.to_string()) + .y_labels(data.len()) + .draw() + .unwrap(); + + for (i, (_, x, y)) in data.into_iter().enumerate() { + let base = i as f64; + + chart + .draw_series(AreaSeries::new( + x.iter().zip(y.iter()).map(|(x, y)| (*x, base + *y / 2.0)), + base, + &DARK_BLUE.mix(0.25), + )) + .unwrap(); + + chart + .draw_series(AreaSeries::new( + x.iter().zip(y.iter()).map(|(x, y)| (*x, base - *y / 2.0)), + base, + &DARK_BLUE.mix(0.25), + )) + .unwrap(); + } +} diff --git a/src/plot/plotters_backend/t_test.rs b/src/plot/plotters_backend/t_test.rs new file mode 100755 index 0000000..741cc9e --- /dev/null +++ b/src/plot/plotters_backend/t_test.rs @@ -0,0 +1,59 @@ +use super::*; +use crate::report::ComparisonData; +use std::path::Path; + +pub(crate) fn t_test( + path: &Path, + title: &str, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + let t = comparison.t_value; + let (xs, ys) = kde::sweep(&comparison.t_distribution, KDE_POINTS, None); + + let x_range = plotters::data::fitting_range(xs.iter()); + let mut y_range = plotters::data::fitting_range(ys.iter()); + y_range.start = 0.0; + y_range.end *= 1.1; + + let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area(); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption(format!("{}: Welch t test", title), (DEFAULT_FONT, 20)) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range.clone()) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Density") + .x_desc("t score") + .draw() + .unwrap(); + + chart + .draw_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + 0.0, + &DARK_BLUE.mix(0.25), + )) + .unwrap() + .label("t distribution") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(t, 0.0), (t, y_range.end)], + DARK_BLUE.filled().stroke_width(2), + ))) + .unwrap() + .label("t statistic") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart.configure_series_labels().draw().unwrap(); +} diff --git a/src/profiler.rs b/src/profiler.rs new file mode 100755 index 0000000..906af5f --- /dev/null +++ b/src/profiler.rs @@ -0,0 +1,28 @@ +//! This module provides an extension trait which allows in-process profilers +//! to be hooked into the `--profile-time` argument at compile-time. Users of +//! out-of-process profilers such as perf don't need to do anything special. + +use std::path::Path; + +/// Extension trait for external crates to implement which provides start/stop +/// hooks when profiling (but not when benchmarking) functions. +pub trait Profiler { + /// This function is called when Criterion.rs starts profiling a particular + /// benchmark. It provides the stringified benchmark ID and + /// a path to a directory where the profiler can store its data. + fn start_profiling(&mut self, benchmark_id: &str, benchmark_dir: &Path); + + /// This function is called after Criterion.rs stops profiling a particular + /// benchmark. The benchmark ID and directory are the same as in the call + /// to `start`, provided for convenience. + fn stop_profiling(&mut self, benchmark_id: &str, benchmark_dir: &Path); +} + +/// Dummy profiler implementation, representing cases where the profiler is +/// an external process (eg. perftools, etc.) which do not require start/stop +/// hooks. This implementation does nothing and is used as the default. +pub struct ExternalProfiler; +impl Profiler for ExternalProfiler { + fn start_profiling(&mut self, _benchmark_id: &str, _benchmark_dir: &Path) {} + fn stop_profiling(&mut self, _benchmark_id: &str, _benchmark_dir: &Path) {} +} diff --git a/src/report.rs b/src/report.rs new file mode 100755 index 0000000..216904f --- /dev/null +++ b/src/report.rs @@ -0,0 +1,880 @@ +use crate::stats::bivariate::regression::Slope; +use crate::stats::bivariate::Data; +use crate::stats::univariate::outliers::tukey::LabeledSample; + +use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates}; +use crate::format; +use crate::measurement::ValueFormatter; +use crate::stats::univariate::Sample; +use crate::stats::Distribution; +use crate::{PlotConfiguration, Throughput}; +use std::cell::Cell; +use std::cmp; +use std::collections::HashSet; +use std::fmt; +use std::io::stdout; +use std::io::Write; +use std::path::{Path, PathBuf}; + +const MAX_DIRECTORY_NAME_LEN: usize = 64; +const MAX_TITLE_LEN: usize = 100; + +pub(crate) struct ComparisonData { + pub p_value: f64, + pub t_distribution: Distribution<f64>, + pub t_value: f64, + pub relative_estimates: ChangeEstimates, + pub relative_distributions: ChangeDistributions, + pub significance_threshold: f64, + pub noise_threshold: f64, + pub base_iter_counts: Vec<f64>, + pub base_sample_times: Vec<f64>, + pub base_avg_times: Vec<f64>, + pub base_estimates: Estimates, +} + +pub(crate) struct MeasurementData<'a> { + pub data: Data<'a, f64, f64>, + pub avg_times: LabeledSample<'a, f64>, + pub absolute_estimates: Estimates, + pub distributions: Distributions, + pub comparison: Option<ComparisonData>, + pub throughput: Option<Throughput>, +} +impl<'a> MeasurementData<'a> { + pub fn iter_counts(&self) -> &Sample<f64> { + self.data.x() + } + + pub fn sample_times(&self) -> &Sample<f64> { + self.data.y() + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ValueType { + Bytes, + Elements, + Value, +} + +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct BenchmarkId { + pub group_id: String, + pub function_id: Option<String>, + pub value_str: Option<String>, + pub throughput: Option<Throughput>, + full_id: String, + directory_name: String, + title: String, +} + +fn truncate_to_character_boundary(s: &mut String, max_len: usize) { + let mut boundary = cmp::min(max_len, s.len()); + while !s.is_char_boundary(boundary) { + boundary -= 1; + } + s.truncate(boundary); +} + +pub fn make_filename_safe(string: &str) -> String { + let mut string = string.replace( + &['?', '"', '/', '\\', '*', '<', '>', ':', '|', '^'][..], + "_", + ); + + // Truncate to last character boundary before max length... + truncate_to_character_boundary(&mut string, MAX_DIRECTORY_NAME_LEN); + + if cfg!(target_os = "windows") { + { + string = string + // On Windows, spaces in the end of the filename are ignored and will be trimmed. + // + // Without trimming ourselves, creating a directory `dir ` will silently create + // `dir` instead, but then operations on files like `dir /file` will fail. + // + // Also note that it's important to do this *after* trimming to MAX_DIRECTORY_NAME_LEN, + // otherwise it can trim again to a name with a trailing space. + .trim_end() + // On Windows, file names are not case-sensitive, so lowercase everything. + .to_lowercase(); + } + } + + string +} + +impl BenchmarkId { + pub fn new( + group_id: String, + function_id: Option<String>, + value_str: Option<String>, + throughput: Option<Throughput>, + ) -> BenchmarkId { + let full_id = match (&function_id, &value_str) { + (&Some(ref func), &Some(ref val)) => format!("{}/{}/{}", group_id, func, val), + (&Some(ref func), &None) => format!("{}/{}", group_id, func), + (&None, &Some(ref val)) => format!("{}/{}", group_id, val), + (&None, &None) => group_id.clone(), + }; + + let mut title = full_id.clone(); + truncate_to_character_boundary(&mut title, MAX_TITLE_LEN); + if title != full_id { + title.push_str("..."); + } + + let directory_name = match (&function_id, &value_str) { + (&Some(ref func), &Some(ref val)) => format!( + "{}/{}/{}", + make_filename_safe(&group_id), + make_filename_safe(func), + make_filename_safe(val) + ), + (&Some(ref func), &None) => format!( + "{}/{}", + make_filename_safe(&group_id), + make_filename_safe(func) + ), + (&None, &Some(ref val)) => format!( + "{}/{}", + make_filename_safe(&group_id), + make_filename_safe(val) + ), + (&None, &None) => make_filename_safe(&group_id), + }; + + BenchmarkId { + group_id, + function_id, + value_str, + throughput, + full_id, + directory_name, + title, + } + } + + pub fn id(&self) -> &str { + &self.full_id + } + + pub fn as_title(&self) -> &str { + &self.title + } + + pub fn as_directory_name(&self) -> &str { + &self.directory_name + } + + pub fn as_number(&self) -> Option<f64> { + match self.throughput { + Some(Throughput::Bytes(n)) | Some(Throughput::Elements(n)) => Some(n as f64), + None => self + .value_str + .as_ref() + .and_then(|string| string.parse::<f64>().ok()), + } + } + + pub fn value_type(&self) -> Option<ValueType> { + match self.throughput { + Some(Throughput::Bytes(_)) => Some(ValueType::Bytes), + Some(Throughput::Elements(_)) => Some(ValueType::Elements), + None => self + .value_str + .as_ref() + .and_then(|string| string.parse::<f64>().ok()) + .map(|_| ValueType::Value), + } + } + + pub fn ensure_directory_name_unique(&mut self, existing_directories: &HashSet<String>) { + if !existing_directories.contains(self.as_directory_name()) { + return; + } + + let mut counter = 2; + loop { + let new_dir_name = format!("{}_{}", self.as_directory_name(), counter); + if !existing_directories.contains(&new_dir_name) { + self.directory_name = new_dir_name; + return; + } + counter += 1; + } + } + + pub fn ensure_title_unique(&mut self, existing_titles: &HashSet<String>) { + if !existing_titles.contains(self.as_title()) { + return; + } + + let mut counter = 2; + loop { + let new_title = format!("{} #{}", self.as_title(), counter); + if !existing_titles.contains(&new_title) { + self.title = new_title; + return; + } + counter += 1; + } + } +} +impl fmt::Display for BenchmarkId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_title()) + } +} +impl fmt::Debug for BenchmarkId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fn format_opt(opt: &Option<String>) -> String { + match *opt { + Some(ref string) => format!("\"{}\"", string), + None => "None".to_owned(), + } + } + + write!( + f, + "BenchmarkId {{ group_id: \"{}\", function_id: {}, value_str: {}, throughput: {:?} }}", + self.group_id, + format_opt(&self.function_id), + format_opt(&self.value_str), + self.throughput, + ) + } +} + +pub struct ReportContext { + pub output_directory: PathBuf, + pub plot_config: PlotConfiguration, +} +impl ReportContext { + pub fn report_path<P: AsRef<Path> + ?Sized>(&self, id: &BenchmarkId, file_name: &P) -> PathBuf { + let mut path = self.output_directory.clone(); + path.push(id.as_directory_name()); + path.push("report"); + path.push(file_name); + path + } +} + +pub(crate) trait Report { + fn test_start(&self, _id: &BenchmarkId, _context: &ReportContext) {} + fn test_pass(&self, _id: &BenchmarkId, _context: &ReportContext) {} + + fn benchmark_start(&self, _id: &BenchmarkId, _context: &ReportContext) {} + fn profile(&self, _id: &BenchmarkId, _context: &ReportContext, _profile_ns: f64) {} + fn warmup(&self, _id: &BenchmarkId, _context: &ReportContext, _warmup_ns: f64) {} + fn terminated(&self, _id: &BenchmarkId, _context: &ReportContext) {} + fn analysis(&self, _id: &BenchmarkId, _context: &ReportContext) {} + fn measurement_start( + &self, + _id: &BenchmarkId, + _context: &ReportContext, + _sample_count: u64, + _estimate_ns: f64, + _iter_count: u64, + ) { + } + fn measurement_complete( + &self, + _id: &BenchmarkId, + _context: &ReportContext, + _measurements: &MeasurementData<'_>, + _formatter: &dyn ValueFormatter, + ) { + } + fn summarize( + &self, + _context: &ReportContext, + _all_ids: &[BenchmarkId], + _formatter: &dyn ValueFormatter, + ) { + } + fn final_summary(&self, _context: &ReportContext) {} + fn group_separator(&self) {} +} + +pub(crate) struct Reports { + reports: Vec<Box<dyn Report>>, +} +impl Reports { + pub fn new(reports: Vec<Box<dyn Report>>) -> Reports { + Reports { reports } + } +} +impl Report for Reports { + fn test_start(&self, id: &BenchmarkId, context: &ReportContext) { + for report in &self.reports { + report.test_start(id, context); + } + } + fn test_pass(&self, id: &BenchmarkId, context: &ReportContext) { + for report in &self.reports { + report.test_pass(id, context); + } + } + + fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext) { + for report in &self.reports { + report.benchmark_start(id, context); + } + } + + fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64) { + for report in &self.reports { + report.profile(id, context, profile_ns); + } + } + + fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64) { + for report in &self.reports { + report.warmup(id, context, warmup_ns); + } + } + + fn terminated(&self, id: &BenchmarkId, context: &ReportContext) { + for report in &self.reports { + report.terminated(id, context); + } + } + + fn analysis(&self, id: &BenchmarkId, context: &ReportContext) { + for report in &self.reports { + report.analysis(id, context); + } + } + + fn measurement_start( + &self, + id: &BenchmarkId, + context: &ReportContext, + sample_count: u64, + estimate_ns: f64, + iter_count: u64, + ) { + for report in &self.reports { + report.measurement_start(id, context, sample_count, estimate_ns, iter_count); + } + } + + fn measurement_complete( + &self, + id: &BenchmarkId, + context: &ReportContext, + measurements: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) { + for report in &self.reports { + report.measurement_complete(id, context, measurements, formatter); + } + } + + fn summarize( + &self, + context: &ReportContext, + all_ids: &[BenchmarkId], + formatter: &dyn ValueFormatter, + ) { + for report in &self.reports { + report.summarize(context, all_ids, formatter); + } + } + + fn final_summary(&self, context: &ReportContext) { + for report in &self.reports { + report.final_summary(context); + } + } + fn group_separator(&self) { + for report in &self.reports { + report.group_separator(); + } + } +} + +pub(crate) struct CliReport { + pub enable_text_overwrite: bool, + pub enable_text_coloring: bool, + pub verbose: bool, + + last_line_len: Cell<usize>, +} +impl CliReport { + pub fn new( + enable_text_overwrite: bool, + enable_text_coloring: bool, + verbose: bool, + ) -> CliReport { + CliReport { + enable_text_overwrite, + enable_text_coloring, + verbose, + + last_line_len: Cell::new(0), + } + } + + fn text_overwrite(&self) { + if self.enable_text_overwrite { + print!("\r"); + for _ in 0..self.last_line_len.get() { + print!(" "); + } + print!("\r"); + } + } + + //Passing a String is the common case here. + #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))] + fn print_overwritable(&self, s: String) { + if self.enable_text_overwrite { + self.last_line_len.set(s.len()); + print!("{}", s); + stdout().flush().unwrap(); + } else { + println!("{}", s); + } + } + + fn green(&self, s: String) -> String { + if self.enable_text_coloring { + format!("\x1B[32m{}\x1B[39m", s) + } else { + s + } + } + + fn yellow(&self, s: String) -> String { + if self.enable_text_coloring { + format!("\x1B[33m{}\x1B[39m", s) + } else { + s + } + } + + fn red(&self, s: String) -> String { + if self.enable_text_coloring { + format!("\x1B[31m{}\x1B[39m", s) + } else { + s + } + } + + fn bold(&self, s: String) -> String { + if self.enable_text_coloring { + format!("\x1B[1m{}\x1B[22m", s) + } else { + s + } + } + + fn faint(&self, s: String) -> String { + if self.enable_text_coloring { + format!("\x1B[2m{}\x1B[22m", s) + } else { + s + } + } + + pub fn outliers(&self, sample: &LabeledSample<'_, f64>) { + let (los, lom, _, him, his) = sample.count(); + let noutliers = los + lom + him + his; + let sample_size = sample.len(); + + if noutliers == 0 { + return; + } + + let percent = |n: usize| 100. * n as f64 / sample_size as f64; + + println!( + "{}", + self.yellow(format!( + "Found {} outliers among {} measurements ({:.2}%)", + noutliers, + sample_size, + percent(noutliers) + )) + ); + + let print = |n, label| { + if n != 0 { + println!(" {} ({:.2}%) {}", n, percent(n), label); + } + }; + + print(los, "low severe"); + print(lom, "low mild"); + print(him, "high mild"); + print(his, "high severe"); + } +} +impl Report for CliReport { + fn test_start(&self, id: &BenchmarkId, _: &ReportContext) { + println!("Testing {}", id); + } + fn test_pass(&self, _: &BenchmarkId, _: &ReportContext) { + println!("Success"); + } + + fn benchmark_start(&self, id: &BenchmarkId, _: &ReportContext) { + self.print_overwritable(format!("Benchmarking {}", id)); + } + + fn profile(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) { + self.text_overwrite(); + self.print_overwritable(format!( + "Benchmarking {}: Profiling for {}", + id, + format::time(warmup_ns) + )); + } + + fn warmup(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) { + self.text_overwrite(); + self.print_overwritable(format!( + "Benchmarking {}: Warming up for {}", + id, + format::time(warmup_ns) + )); + } + + fn terminated(&self, id: &BenchmarkId, _: &ReportContext) { + self.text_overwrite(); + println!("Benchmarking {}: Complete (Analysis Disabled)", id); + } + + fn analysis(&self, id: &BenchmarkId, _: &ReportContext) { + self.text_overwrite(); + self.print_overwritable(format!("Benchmarking {}: Analyzing", id)); + } + + fn measurement_start( + &self, + id: &BenchmarkId, + _: &ReportContext, + sample_count: u64, + estimate_ns: f64, + iter_count: u64, + ) { + self.text_overwrite(); + let iter_string = if self.verbose { + format!("{} iterations", iter_count) + } else { + format::iter_count(iter_count) + }; + + self.print_overwritable(format!( + "Benchmarking {}: Collecting {} samples in estimated {} ({})", + id, + sample_count, + format::time(estimate_ns), + iter_string + )); + } + + fn measurement_complete( + &self, + id: &BenchmarkId, + _: &ReportContext, + meas: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) { + self.text_overwrite(); + + let typical_estimate = &meas.absolute_estimates.typical(); + + { + let mut id = id.as_title().to_owned(); + + if id.len() > 23 { + println!("{}", self.green(id.clone())); + id.clear(); + } + let id_len = id.len(); + + println!( + "{}{}time: [{} {} {}]", + self.green(id), + " ".repeat(24 - id_len), + self.faint( + formatter.format_value(typical_estimate.confidence_interval.lower_bound) + ), + self.bold(formatter.format_value(typical_estimate.point_estimate)), + self.faint( + formatter.format_value(typical_estimate.confidence_interval.upper_bound) + ) + ); + } + + if let Some(ref throughput) = meas.throughput { + println!( + "{}thrpt: [{} {} {}]", + " ".repeat(24), + self.faint(formatter.format_throughput( + throughput, + typical_estimate.confidence_interval.upper_bound + )), + self.bold(formatter.format_throughput(throughput, typical_estimate.point_estimate)), + self.faint(formatter.format_throughput( + throughput, + typical_estimate.confidence_interval.lower_bound + )), + ) + } + + if let Some(ref comp) = meas.comparison { + let different_mean = comp.p_value < comp.significance_threshold; + let mean_est = &comp.relative_estimates.mean; + let point_estimate = mean_est.point_estimate; + let mut point_estimate_str = format::change(point_estimate, true); + // The change in throughput is related to the change in timing. Reducing the timing by + // 50% increases the througput by 100%. + let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0; + let mut thrpt_point_estimate_str = + format::change(to_thrpt_estimate(point_estimate), true); + let explanation_str: String; + + if !different_mean { + explanation_str = "No change in performance detected.".to_owned(); + } else { + let comparison = compare_to_threshold(&mean_est, comp.noise_threshold); + match comparison { + ComparisonResult::Improved => { + point_estimate_str = self.green(self.bold(point_estimate_str)); + thrpt_point_estimate_str = self.green(self.bold(thrpt_point_estimate_str)); + explanation_str = + format!("Performance has {}.", self.green("improved".to_owned())); + } + ComparisonResult::Regressed => { + point_estimate_str = self.red(self.bold(point_estimate_str)); + thrpt_point_estimate_str = self.red(self.bold(thrpt_point_estimate_str)); + explanation_str = + format!("Performance has {}.", self.red("regressed".to_owned())); + } + ComparisonResult::NonSignificant => { + explanation_str = "Change within noise threshold.".to_owned(); + } + } + } + + if meas.throughput.is_some() { + println!("{}change:", " ".repeat(17)); + + println!( + "{}time: [{} {} {}] (p = {:.2} {} {:.2})", + " ".repeat(24), + self.faint(format::change( + mean_est.confidence_interval.lower_bound, + true + )), + point_estimate_str, + self.faint(format::change( + mean_est.confidence_interval.upper_bound, + true + )), + comp.p_value, + if different_mean { "<" } else { ">" }, + comp.significance_threshold + ); + println!( + "{}thrpt: [{} {} {}]", + " ".repeat(24), + self.faint(format::change( + to_thrpt_estimate(mean_est.confidence_interval.upper_bound), + true + )), + thrpt_point_estimate_str, + self.faint(format::change( + to_thrpt_estimate(mean_est.confidence_interval.lower_bound), + true + )), + ); + } else { + println!( + "{}change: [{} {} {}] (p = {:.2} {} {:.2})", + " ".repeat(24), + self.faint(format::change( + mean_est.confidence_interval.lower_bound, + true + )), + point_estimate_str, + self.faint(format::change( + mean_est.confidence_interval.upper_bound, + true + )), + comp.p_value, + if different_mean { "<" } else { ">" }, + comp.significance_threshold + ); + } + + println!("{}{}", " ".repeat(24), explanation_str); + } + + self.outliers(&meas.avg_times); + + if self.verbose { + let format_short_estimate = |estimate: &Estimate| -> String { + format!( + "[{} {}]", + formatter.format_value(estimate.confidence_interval.lower_bound), + formatter.format_value(estimate.confidence_interval.upper_bound) + ) + }; + + let data = &meas.data; + if let Some(slope_estimate) = meas.absolute_estimates.slope.as_ref() { + println!( + "{:<7}{} {:<15}[{:0.7} {:0.7}]", + "slope", + format_short_estimate(slope_estimate), + "R^2", + Slope(slope_estimate.confidence_interval.lower_bound).r_squared(data), + Slope(slope_estimate.confidence_interval.upper_bound).r_squared(data), + ); + } + println!( + "{:<7}{} {:<15}{}", + "mean", + format_short_estimate(&meas.absolute_estimates.mean), + "std. dev.", + format_short_estimate(&meas.absolute_estimates.std_dev), + ); + println!( + "{:<7}{} {:<15}{}", + "median", + format_short_estimate(&meas.absolute_estimates.median), + "med. abs. dev.", + format_short_estimate(&meas.absolute_estimates.median_abs_dev), + ); + } + } + + fn group_separator(&self) { + println!(); + } +} + +pub struct BencherReport; +impl Report for BencherReport { + fn measurement_start( + &self, + id: &BenchmarkId, + _context: &ReportContext, + _sample_count: u64, + _estimate_ns: f64, + _iter_count: u64, + ) { + print!("test {} ... ", id); + } + + fn measurement_complete( + &self, + _id: &BenchmarkId, + _: &ReportContext, + meas: &MeasurementData<'_>, + formatter: &dyn ValueFormatter, + ) { + let mut values = [ + meas.absolute_estimates.median.point_estimate, + meas.absolute_estimates.std_dev.point_estimate, + ]; + let unit = formatter.scale_for_machines(&mut values); + + println!( + "bench: {:>11} {}/iter (+/- {})", + format::integer(values[0]), + unit, + format::integer(values[1]) + ); + } + + fn group_separator(&self) { + println!(); + } +} + +enum ComparisonResult { + Improved, + Regressed, + NonSignificant, +} + +fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult { + let ci = &estimate.confidence_interval; + let lb = ci.lower_bound; + let ub = ci.upper_bound; + + if lb < -noise && ub < -noise { + ComparisonResult::Improved + } else if lb > noise && ub > noise { + ComparisonResult::Regressed + } else { + ComparisonResult::NonSignificant + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_make_filename_safe_replaces_characters() { + let input = "?/\\*\""; + let safe = make_filename_safe(input); + assert_eq!("_____", &safe); + } + + #[test] + fn test_make_filename_safe_truncates_long_strings() { + let input = "this is a very long string. it is too long to be safe as a directory name, and so it needs to be truncated. what a long string this is."; + let safe = make_filename_safe(input); + assert!(input.len() > MAX_DIRECTORY_NAME_LEN); + assert_eq!(&input[0..MAX_DIRECTORY_NAME_LEN], &safe); + } + + #[test] + fn test_make_filename_safe_respects_character_boundaries() { + let input = "✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓"; + let safe = make_filename_safe(input); + assert!(safe.len() < MAX_DIRECTORY_NAME_LEN); + } + + #[test] + fn test_benchmark_id_make_directory_name_unique() { + let existing_id = BenchmarkId::new( + "group".to_owned(), + Some("function".to_owned()), + Some("value".to_owned()), + None, + ); + let mut directories = HashSet::new(); + directories.insert(existing_id.as_directory_name().to_owned()); + + let mut new_id = existing_id.clone(); + new_id.ensure_directory_name_unique(&directories); + assert_eq!("group/function/value_2", new_id.as_directory_name()); + directories.insert(new_id.as_directory_name().to_owned()); + + new_id = existing_id.clone(); + new_id.ensure_directory_name_unique(&directories); + assert_eq!("group/function/value_3", new_id.as_directory_name()); + directories.insert(new_id.as_directory_name().to_owned()); + } + #[test] + fn test_benchmark_id_make_long_directory_name_unique() { + let long_name = (0..MAX_DIRECTORY_NAME_LEN).map(|_| 'a').collect::<String>(); + let existing_id = BenchmarkId::new(long_name, None, None, None); + let mut directories = HashSet::new(); + directories.insert(existing_id.as_directory_name().to_owned()); + + let mut new_id = existing_id.clone(); + new_id.ensure_directory_name_unique(&directories); + assert_ne!(existing_id.as_directory_name(), new_id.as_directory_name()); + } +} diff --git a/src/routine.rs b/src/routine.rs new file mode 100755 index 0000000..a54cdd6 --- /dev/null +++ b/src/routine.rs @@ -0,0 +1,240 @@ +use crate::benchmark::BenchmarkConfig; +use crate::connection::OutgoingMessage; +use crate::measurement::Measurement; +use crate::report::{BenchmarkId, ReportContext}; +use crate::{ActualSamplingMode, Bencher, Criterion, DurationExt}; +use std::marker::PhantomData; +use std::time::Duration; + +/// PRIVATE +pub(crate) trait Routine<M: Measurement, T: ?Sized> { + /// PRIVATE + fn bench(&mut self, m: &M, iters: &[u64], parameter: &T) -> Vec<f64>; + /// PRIVATE + fn warm_up(&mut self, m: &M, how_long: Duration, parameter: &T) -> (u64, u64); + + /// PRIVATE + fn test(&mut self, m: &M, parameter: &T) { + self.bench(m, &[1u64], parameter); + } + + /// Iterates the benchmarked function for a fixed length of time, but takes no measurements. + /// This keeps the overall benchmark suite runtime constant-ish even when running under a + /// profiler with an unknown amount of overhead. Since no measurements are taken, it also + /// reduces the amount of time the execution spends in Criterion.rs code, which should help + /// show the performance of the benchmarked code more clearly as well. + fn profile( + &mut self, + measurement: &M, + id: &BenchmarkId, + criterion: &Criterion<M>, + report_context: &ReportContext, + time: Duration, + parameter: &T, + ) { + criterion + .report + .profile(id, report_context, time.to_nanos() as f64); + + let mut profile_path = report_context.output_directory.clone(); + if (*crate::CARGO_CRITERION_CONNECTION).is_some() { + // If connected to cargo-criterion, generate a cargo-criterion-style path. + // This is kind of a hack. + profile_path.push("profile"); + profile_path.push(id.as_directory_name()); + } else { + profile_path.push(id.as_directory_name()); + profile_path.push("profile"); + } + criterion + .profiler + .borrow_mut() + .start_profiling(id.id(), &profile_path); + + let time = time.to_nanos(); + + // TODO: Some profilers will show the two batches of iterations as + // being different code-paths even though they aren't really. + + // Get the warmup time for one second + let (wu_elapsed, wu_iters) = self.warm_up(measurement, Duration::from_secs(1), parameter); + if wu_elapsed < time { + // Initial guess for the mean execution time + let met = wu_elapsed as f64 / wu_iters as f64; + + // Guess how many iterations will be required for the remaining time + let remaining = (time - wu_elapsed) as f64; + + let iters = remaining / met; + let iters = iters as u64; + + self.bench(measurement, &[iters], parameter); + } + + criterion + .profiler + .borrow_mut() + .stop_profiling(id.id(), &profile_path); + + criterion.report.terminated(id, report_context); + } + + fn sample( + &mut self, + measurement: &M, + id: &BenchmarkId, + config: &BenchmarkConfig, + criterion: &Criterion<M>, + report_context: &ReportContext, + parameter: &T, + ) -> (ActualSamplingMode, Box<[f64]>, Box<[f64]>) { + let wu = config.warm_up_time; + let m_ns = config.measurement_time.to_nanos(); + + criterion + .report + .warmup(id, report_context, wu.to_nanos() as f64); + + if let Some(conn) = &criterion.connection { + conn.send(&OutgoingMessage::Warmup { + id: id.into(), + nanos: wu.to_nanos() as f64, + }) + .unwrap(); + } + + let (wu_elapsed, wu_iters) = self.warm_up(measurement, wu, parameter); + if crate::debug_enabled() { + println!( + "\nCompleted {} iterations in {} nanoseconds, estimated execution time is {} ns", + wu_iters, + wu_elapsed, + wu_elapsed as f64 / wu_iters as f64 + ); + } + + // Initial guess for the mean execution time + let met = wu_elapsed as f64 / wu_iters as f64; + + let n = config.sample_size as u64; + + let actual_sampling_mode = config + .sampling_mode + .choose_sampling_mode(met, n, m_ns as f64); + + let m_iters = actual_sampling_mode.iteration_counts(met, n, &config.measurement_time); + + let expected_ns = m_iters + .iter() + .copied() + .map(|count| count as f64 * met) + .sum(); + + criterion.report.measurement_start( + id, + report_context, + n, + expected_ns, + m_iters.iter().sum(), + ); + + if let Some(conn) = &criterion.connection { + conn.send(&OutgoingMessage::MeasurementStart { + id: id.into(), + sample_count: n, + estimate_ns: expected_ns, + iter_count: m_iters.iter().sum(), + }) + .unwrap(); + } + + let m_elapsed = self.bench(measurement, &m_iters, parameter); + + let m_iters_f: Vec<f64> = m_iters.iter().map(|&x| x as f64).collect(); + + ( + actual_sampling_mode, + m_iters_f.into_boxed_slice(), + m_elapsed.into_boxed_slice(), + ) + } +} + +pub struct Function<M: Measurement, F, T> +where + F: FnMut(&mut Bencher<'_, M>, &T), + T: ?Sized, +{ + f: F, + // TODO: Is there some way to remove these? + _phantom: PhantomData<T>, + _phamtom2: PhantomData<M>, +} +impl<M: Measurement, F, T> Function<M, F, T> +where + F: FnMut(&mut Bencher<'_, M>, &T), + T: ?Sized, +{ + pub fn new(f: F) -> Function<M, F, T> { + Function { + f, + _phantom: PhantomData, + _phamtom2: PhantomData, + } + } +} + +impl<M: Measurement, F, T> Routine<M, T> for Function<M, F, T> +where + F: FnMut(&mut Bencher<'_, M>, &T), + T: ?Sized, +{ + fn bench(&mut self, m: &M, iters: &[u64], parameter: &T) -> Vec<f64> { + let f = &mut self.f; + + let mut b = Bencher { + iterated: false, + iters: 0, + value: m.zero(), + measurement: m, + elapsed_time: Duration::from_millis(0), + }; + + iters + .iter() + .map(|iters| { + b.iters = *iters; + (*f)(&mut b, parameter); + b.assert_iterated(); + m.to_f64(&b.value) + }) + .collect() + } + + fn warm_up(&mut self, m: &M, how_long: Duration, parameter: &T) -> (u64, u64) { + let f = &mut self.f; + let mut b = Bencher { + iterated: false, + iters: 1, + value: m.zero(), + measurement: m, + elapsed_time: Duration::from_millis(0), + }; + + let mut total_iters = 0; + let mut elapsed_time = Duration::from_millis(0); + loop { + (*f)(&mut b, parameter); + + b.assert_iterated(); + + total_iters += b.iters; + elapsed_time += b.elapsed_time; + if elapsed_time > how_long { + return (elapsed_time.to_nanos(), total_iters); + } + + b.iters *= 2; + } + } +} diff --git a/src/stats/bivariate/bootstrap.rs b/src/stats/bivariate/bootstrap.rs new file mode 100755 index 0000000..8fe8ede --- /dev/null +++ b/src/stats/bivariate/bootstrap.rs @@ -0,0 +1,83 @@ +#[cfg(test)] +macro_rules! test { + ($ty:ident) => { + mod $ty { + use quickcheck::TestResult; + use quickcheck::quickcheck; + use approx::relative_eq; + + use crate::stats::bivariate::regression::Slope; + use crate::stats::bivariate::Data; + + quickcheck! { + fn means(size: usize, start: usize, + offset: usize, nresamples: usize) -> TestResult { + if let Some(x) = crate::stats::test::vec::<$ty>(size, start) { + let y = crate::stats::test::vec::<$ty>(size + offset, start + offset).unwrap(); + let data = Data::new(&x[start..], &y[start+offset..]); + + let (x_means, y_means) = if nresamples > 0 { + data.bootstrap(nresamples, |d| (d.x().mean(), d.y().mean())) + } else { + return TestResult::discard(); + }; + + let x_min = data.x().min(); + let x_max = data.x().max(); + let y_min = data.y().min(); + let y_max = data.y().max(); + + TestResult::from_bool( + // Computed the correct number of resamples + x_means.len() == nresamples && + y_means.len() == nresamples && + // No uninitialized values + x_means.iter().all(|&x| { + (x > x_min || relative_eq!(x, x_min)) && + (x < x_max || relative_eq!(x, x_max)) + }) && + y_means.iter().all(|&y| { + (y > y_min || relative_eq!(y, y_min)) && + (y < y_max || relative_eq!(y, y_max)) + }) + ) + } else { + TestResult::discard() + } + } + } + + quickcheck! { + fn slope(size: usize, start: usize, + offset: usize, nresamples: usize) -> TestResult { + if let Some(x) = crate::stats::test::vec::<$ty>(size, start) { + let y = crate::stats::test::vec::<$ty>(size + offset, start + offset).unwrap(); + let data = Data::new(&x[start..], &y[start+offset..]); + + let slopes = if nresamples > 0 { + data.bootstrap(nresamples, |d| (Slope::fit(&d),)).0 + } else { + return TestResult::discard(); + }; + + TestResult::from_bool( + // Computed the correct number of resamples + slopes.len() == nresamples && + // No uninitialized values + slopes.iter().all(|s| s.0 > 0.) + ) + } else { + TestResult::discard() + } + } + } + + } + }; +} + +#[cfg(test)] +mod test { + test!(f32); + test!(f64); +} diff --git a/src/stats/bivariate/mod.rs b/src/stats/bivariate/mod.rs new file mode 100755 index 0000000..b233b60 --- /dev/null +++ b/src/stats/bivariate/mod.rs @@ -0,0 +1,131 @@ +//! Bivariate analysis + +mod bootstrap; +pub mod regression; +mod resamples; + +use crate::stats::bivariate::resamples::Resamples; +use crate::stats::float::Float; +use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; +use crate::stats::univariate::Sample; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +/// Bivariate `(X, Y)` data +/// +/// Invariants: +/// +/// - No `NaN`s in the data +/// - At least two data points in the set +pub struct Data<'a, X, Y>(&'a [X], &'a [Y]); + +impl<'a, X, Y> Copy for Data<'a, X, Y> {} + +#[cfg_attr(feature = "cargo-clippy", allow(clippy::expl_impl_clone_on_copy))] +impl<'a, X, Y> Clone for Data<'a, X, Y> { + fn clone(&self) -> Data<'a, X, Y> { + *self + } +} + +impl<'a, X, Y> Data<'a, X, Y> { + /// Returns the length of the data set + pub fn len(&self) -> usize { + self.0.len() + } + + /// Iterate over the data set + pub fn iter(&self) -> Pairs<'a, X, Y> { + Pairs { + data: *self, + state: 0, + } + } +} + +impl<'a, X, Y> Data<'a, X, Y> +where + X: Float, + Y: Float, +{ + /// Creates a new data set from two existing slices + pub fn new(xs: &'a [X], ys: &'a [Y]) -> Data<'a, X, Y> { + assert!( + xs.len() == ys.len() + && xs.len() > 1 + && xs.iter().all(|x| !x.is_nan()) + && ys.iter().all(|y| !y.is_nan()) + ); + + Data(xs, ys) + } + + // TODO Remove the `T` parameter in favor of `S::Output` + /// Returns the bootstrap distributions of the parameters estimated by the `statistic` + /// + /// - Multi-threaded + /// - Time: `O(nresamples)` + /// - Memory: `O(nresamples)` + pub fn bootstrap<T, S>(&self, nresamples: usize, statistic: S) -> T::Distributions + where + S: Fn(Data<X, Y>) -> T + Sync, + T: Tuple + Send, + T::Distributions: Send, + T::Builder: Send, + { + (0..nresamples) + .into_par_iter() + .map_init( + || Resamples::new(*self), + |resamples, _| statistic(resamples.next()), + ) + .fold( + || T::Builder::new(0), + |mut sub_distributions, sample| { + sub_distributions.push(sample); + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() + } + + /// Returns a view into the `X` data + pub fn x(&self) -> &'a Sample<X> { + Sample::new(&self.0) + } + + /// Returns a view into the `Y` data + pub fn y(&self) -> &'a Sample<Y> { + Sample::new(&self.1) + } +} + +/// Iterator over `Data` +pub struct Pairs<'a, X: 'a, Y: 'a> { + data: Data<'a, X, Y>, + state: usize, +} + +impl<'a, X, Y> Iterator for Pairs<'a, X, Y> { + type Item = (&'a X, &'a Y); + + fn next(&mut self) -> Option<(&'a X, &'a Y)> { + if self.state < self.data.len() { + let i = self.state; + self.state += 1; + + // This is safe because i will always be < self.data.{0,1}.len() + debug_assert!(i < self.data.0.len()); + debug_assert!(i < self.data.1.len()); + unsafe { Some((self.data.0.get_unchecked(i), self.data.1.get_unchecked(i))) } + } else { + None + } + } +} diff --git a/src/stats/bivariate/regression.rs b/src/stats/bivariate/regression.rs new file mode 100755 index 0000000..f09443f --- /dev/null +++ b/src/stats/bivariate/regression.rs @@ -0,0 +1,53 @@ +//! Regression analysis + +use crate::stats::bivariate::Data; +use crate::stats::float::Float; + +/// A straight line that passes through the origin `y = m * x` +#[derive(Clone, Copy)] +pub struct Slope<A>(pub A) +where + A: Float; + +impl<A> Slope<A> +where + A: Float, +{ + /// Fits the data to a straight line that passes through the origin using ordinary least + /// squares + /// + /// - Time: `O(length)` + pub fn fit(data: &Data<'_, A, A>) -> Slope<A> { + let xs = data.0; + let ys = data.1; + + let xy = crate::stats::dot(xs, ys); + let x2 = crate::stats::dot(xs, xs); + + Slope(xy / x2) + } + + /// Computes the goodness of fit (coefficient of determination) for this data set + /// + /// - Time: `O(length)` + pub fn r_squared(&self, data: &Data<'_, A, A>) -> A { + let _0 = A::cast(0); + let _1 = A::cast(1); + let m = self.0; + let xs = data.0; + let ys = data.1; + + let n = A::cast(xs.len()); + let y_bar = crate::stats::sum(ys) / n; + + let mut ss_res = _0; + let mut ss_tot = _0; + + for (&x, &y) in data.iter() { + ss_res = ss_res + (y - m * x).powi(2); + ss_tot = ss_res + (y - y_bar).powi(2); + } + + _1 - ss_res / ss_tot + } +} diff --git a/src/stats/bivariate/resamples.rs b/src/stats/bivariate/resamples.rs new file mode 100755 index 0000000..e254dc7 --- /dev/null +++ b/src/stats/bivariate/resamples.rs @@ -0,0 +1,61 @@ +use crate::stats::bivariate::Data; +use crate::stats::float::Float; +use crate::stats::rand_util::{new_rng, Rng}; + +pub struct Resamples<'a, X, Y> +where + X: 'a + Float, + Y: 'a + Float, +{ + rng: Rng, + data: (&'a [X], &'a [Y]), + stage: Option<(Vec<X>, Vec<Y>)>, +} + +#[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))] +impl<'a, X, Y> Resamples<'a, X, Y> +where + X: 'a + Float, + Y: 'a + Float, +{ + pub fn new(data: Data<'a, X, Y>) -> Resamples<'a, X, Y> { + Resamples { + rng: new_rng(), + data: (data.x(), data.y()), + stage: None, + } + } + + pub fn next(&mut self) -> Data<'_, X, Y> { + let n = self.data.0.len(); + + match self.stage { + None => { + let mut stage = (Vec::with_capacity(n), Vec::with_capacity(n)); + + for _ in 0..n { + let i = self.rng.rand_range(0u64..(self.data.0.len() as u64)) as usize; + + stage.0.push(self.data.0[i]); + stage.1.push(self.data.1[i]); + } + + self.stage = Some(stage); + } + Some(ref mut stage) => { + for i in 0..n { + let j = self.rng.rand_range(0u64..(self.data.0.len() as u64)) as usize; + + stage.0[i] = self.data.0[j]; + stage.1[i] = self.data.1[j]; + } + } + } + + if let Some((ref x, ref y)) = self.stage { + Data(x, y) + } else { + unreachable!(); + } + } +} diff --git a/src/stats/float.rs b/src/stats/float.rs new file mode 100755 index 0000000..b7748dd --- /dev/null +++ b/src/stats/float.rs @@ -0,0 +1,15 @@ +//! Float trait + +use cast::From; +use num_traits::float; + +/// This is an extension of `num_traits::float::Float` that adds safe +/// casting and Sync + Send. Once `num_traits` has these features this +/// can be removed. +pub trait Float: + float::Float + From<usize, Output = Self> + From<f32, Output = Self> + Sync + Send +{ +} + +impl Float for f32 {} +impl Float for f64 {} diff --git a/src/stats/mod.rs b/src/stats/mod.rs new file mode 100755 index 0000000..4f926de --- /dev/null +++ b/src/stats/mod.rs @@ -0,0 +1,112 @@ +//! [Criterion]'s statistics library. +//! +//! [Criterion]: https://github.com/bheisler/criterion.rs +//! +//! **WARNING** This library is criterion's implementation detail and there no plans to stabilize +//! it. In other words, the API may break at any time without notice. + +#[cfg(test)] +mod test; + +pub mod bivariate; +pub mod tuple; +pub mod univariate; + +mod float; +mod rand_util; + +use std::mem; +use std::ops::Deref; + +use crate::stats::float::Float; +use crate::stats::univariate::Sample; + +/// The bootstrap distribution of some parameter +#[derive(Clone)] +pub struct Distribution<A>(Box<[A]>); + +impl<A> Distribution<A> +where + A: Float, +{ + /// Create a distribution from the given values + pub fn from(values: Box<[A]>) -> Distribution<A> { + Distribution(values) + } + + /// Computes the confidence interval of the population parameter using percentiles + /// + /// # Panics + /// + /// Panics if the `confidence_level` is not in the `(0, 1)` range. + pub fn confidence_interval(&self, confidence_level: A) -> (A, A) + where + usize: cast::From<A, Output = Result<usize, cast::Error>>, + { + let _0 = A::cast(0); + let _1 = A::cast(1); + let _50 = A::cast(50); + + assert!(confidence_level > _0 && confidence_level < _1); + + let percentiles = self.percentiles(); + + // FIXME(privacy) this should use the `at_unchecked()` method + ( + percentiles.at(_50 * (_1 - confidence_level)), + percentiles.at(_50 * (_1 + confidence_level)), + ) + } + + /// Computes the "likelihood" of seeing the value `t` or "more extreme" values in the + /// distribution. + pub fn p_value(&self, t: A, tails: &Tails) -> A { + use std::cmp; + + let n = self.0.len(); + let hits = self.0.iter().filter(|&&x| x < t).count(); + + let tails = A::cast(match *tails { + Tails::One => 1, + Tails::Two => 2, + }); + + A::cast(cmp::min(hits, n - hits)) / A::cast(n) * tails + } +} + +impl<A> Deref for Distribution<A> { + type Target = Sample<A>; + + fn deref(&self) -> &Sample<A> { + let slice: &[_] = &self.0; + + unsafe { mem::transmute(slice) } + } +} + +/// Number of tails for significance testing +pub enum Tails { + /// One tailed test + One, + /// Two tailed test + Two, +} + +fn dot<A>(xs: &[A], ys: &[A]) -> A +where + A: Float, +{ + xs.iter() + .zip(ys) + .fold(A::cast(0), |acc, (&x, &y)| acc + x * y) +} + +fn sum<A>(xs: &[A]) -> A +where + A: Float, +{ + use std::ops::Add; + + xs.iter().cloned().fold(A::cast(0), Add::add) +} diff --git a/src/stats/rand_util.rs b/src/stats/rand_util.rs new file mode 100755 index 0000000..ed374cf --- /dev/null +++ b/src/stats/rand_util.rs @@ -0,0 +1,21 @@ +use oorandom::Rand64; +use std::cell::RefCell; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub type Rng = Rand64; + +thread_local! { + static SEED_RAND: RefCell<Rand64> = RefCell::new(Rand64::new( + SystemTime::now().duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_millis() + )); +} + +pub fn new_rng() -> Rng { + SEED_RAND.with(|r| { + let mut r = r.borrow_mut(); + let seed = ((r.rand_u64() as u128) << 64) | (r.rand_u64() as u128); + Rand64::new(seed) + }) +} diff --git a/src/stats/test.rs b/src/stats/test.rs new file mode 100755 index 0000000..9e13f30 --- /dev/null +++ b/src/stats/test.rs @@ -0,0 +1,16 @@ +use rand::distributions::{Distribution, Standard}; +use rand::prelude::*; +use rand::rngs::StdRng; + +pub fn vec<T>(size: usize, start: usize) -> Option<Vec<T>> +where + Standard: Distribution<T>, +{ + if size > start + 2 { + let mut rng = StdRng::from_entropy(); + + Some((0..size).map(|_| rng.gen()).collect()) + } else { + None + } +} diff --git a/src/stats/tuple.rs b/src/stats/tuple.rs new file mode 100755 index 0000000..1c07515 --- /dev/null +++ b/src/stats/tuple.rs @@ -0,0 +1,253 @@ +//! Helper traits for tupling/untupling + +use crate::stats::Distribution; + +/// Any tuple: `(A, B, ..)` +pub trait Tuple: Sized { + /// A tuple of distributions associated with this tuple + type Distributions: TupledDistributions<Item = Self>; + + /// A tuple of vectors associated with this tuple + type Builder: TupledDistributionsBuilder<Item = Self>; +} + +/// A tuple of distributions: `(Distribution<A>, Distribution<B>, ..)` +pub trait TupledDistributions: Sized { + /// A tuple that can be pushed/inserted into the tupled distributions + type Item: Tuple<Distributions = Self>; +} + +/// A tuple of vecs used to build distributions. +pub trait TupledDistributionsBuilder: Sized { + /// A tuple that can be pushed/inserted into the tupled distributions + type Item: Tuple<Builder = Self>; + + /// Creates a new tuple of vecs + fn new(size: usize) -> Self; + + /// Push one element into each of the vecs + fn push(&mut self, tuple: Self::Item); + + /// Append one tuple of vecs to this one, leaving the vecs in the other tuple empty + fn extend(&mut self, other: &mut Self); + + /// Convert the tuple of vectors into a tuple of distributions + fn complete(self) -> <Self::Item as Tuple>::Distributions; +} + +impl<A> Tuple for (A,) +where + A: Copy, +{ + type Distributions = (Distribution<A>,); + type Builder = (Vec<A>,); +} + +impl<A> TupledDistributions for (Distribution<A>,) +where + A: Copy, +{ + type Item = (A,); +} +impl<A> TupledDistributionsBuilder for (Vec<A>,) +where + A: Copy, +{ + type Item = (A,); + + fn new(size: usize) -> (Vec<A>,) { + (Vec::with_capacity(size),) + } + + fn push(&mut self, tuple: (A,)) { + (self.0).push(tuple.0); + } + + fn extend(&mut self, other: &mut (Vec<A>,)) { + (self.0).append(&mut other.0); + } + + fn complete(self) -> (Distribution<A>,) { + (Distribution(self.0.into_boxed_slice()),) + } +} + +impl<A, B> Tuple for (A, B) +where + A: Copy, + B: Copy, +{ + type Distributions = (Distribution<A>, Distribution<B>); + type Builder = (Vec<A>, Vec<B>); +} + +impl<A, B> TupledDistributions for (Distribution<A>, Distribution<B>) +where + A: Copy, + B: Copy, +{ + type Item = (A, B); +} +impl<A, B> TupledDistributionsBuilder for (Vec<A>, Vec<B>) +where + A: Copy, + B: Copy, +{ + type Item = (A, B); + + fn new(size: usize) -> (Vec<A>, Vec<B>) { + (Vec::with_capacity(size), Vec::with_capacity(size)) + } + + fn push(&mut self, tuple: (A, B)) { + (self.0).push(tuple.0); + (self.1).push(tuple.1); + } + + fn extend(&mut self, other: &mut (Vec<A>, Vec<B>)) { + (self.0).append(&mut other.0); + (self.1).append(&mut other.1); + } + + fn complete(self) -> (Distribution<A>, Distribution<B>) { + ( + Distribution(self.0.into_boxed_slice()), + Distribution(self.1.into_boxed_slice()), + ) + } +} + +impl<A, B, C> Tuple for (A, B, C) +where + A: Copy, + B: Copy, + C: Copy, +{ + type Distributions = (Distribution<A>, Distribution<B>, Distribution<C>); + type Builder = (Vec<A>, Vec<B>, Vec<C>); +} + +impl<A, B, C> TupledDistributions for (Distribution<A>, Distribution<B>, Distribution<C>) +where + A: Copy, + B: Copy, + C: Copy, +{ + type Item = (A, B, C); +} +impl<A, B, C> TupledDistributionsBuilder for (Vec<A>, Vec<B>, Vec<C>) +where + A: Copy, + B: Copy, + C: Copy, +{ + type Item = (A, B, C); + + fn new(size: usize) -> (Vec<A>, Vec<B>, Vec<C>) { + ( + Vec::with_capacity(size), + Vec::with_capacity(size), + Vec::with_capacity(size), + ) + } + + fn push(&mut self, tuple: (A, B, C)) { + (self.0).push(tuple.0); + (self.1).push(tuple.1); + (self.2).push(tuple.2); + } + + fn extend(&mut self, other: &mut (Vec<A>, Vec<B>, Vec<C>)) { + (self.0).append(&mut other.0); + (self.1).append(&mut other.1); + (self.2).append(&mut other.2); + } + + fn complete(self) -> (Distribution<A>, Distribution<B>, Distribution<C>) { + ( + Distribution(self.0.into_boxed_slice()), + Distribution(self.1.into_boxed_slice()), + Distribution(self.2.into_boxed_slice()), + ) + } +} + +impl<A, B, C, D> Tuple for (A, B, C, D) +where + A: Copy, + B: Copy, + C: Copy, + D: Copy, +{ + type Distributions = ( + Distribution<A>, + Distribution<B>, + Distribution<C>, + Distribution<D>, + ); + type Builder = (Vec<A>, Vec<B>, Vec<C>, Vec<D>); +} + +impl<A, B, C, D> TupledDistributions + for ( + Distribution<A>, + Distribution<B>, + Distribution<C>, + Distribution<D>, + ) +where + A: Copy, + B: Copy, + C: Copy, + D: Copy, +{ + type Item = (A, B, C, D); +} +impl<A, B, C, D> TupledDistributionsBuilder for (Vec<A>, Vec<B>, Vec<C>, Vec<D>) +where + A: Copy, + B: Copy, + C: Copy, + D: Copy, +{ + type Item = (A, B, C, D); + + fn new(size: usize) -> (Vec<A>, Vec<B>, Vec<C>, Vec<D>) { + ( + Vec::with_capacity(size), + Vec::with_capacity(size), + Vec::with_capacity(size), + Vec::with_capacity(size), + ) + } + + fn push(&mut self, tuple: (A, B, C, D)) { + (self.0).push(tuple.0); + (self.1).push(tuple.1); + (self.2).push(tuple.2); + (self.3).push(tuple.3); + } + + fn extend(&mut self, other: &mut (Vec<A>, Vec<B>, Vec<C>, Vec<D>)) { + (self.0).append(&mut other.0); + (self.1).append(&mut other.1); + (self.2).append(&mut other.2); + (self.3).append(&mut other.3); + } + + fn complete( + self, + ) -> ( + Distribution<A>, + Distribution<B>, + Distribution<C>, + Distribution<D>, + ) { + ( + Distribution(self.0.into_boxed_slice()), + Distribution(self.1.into_boxed_slice()), + Distribution(self.2.into_boxed_slice()), + Distribution(self.3.into_boxed_slice()), + ) + } +} diff --git a/src/stats/univariate/bootstrap.rs b/src/stats/univariate/bootstrap.rs new file mode 100755 index 0000000..dbb52f5 --- /dev/null +++ b/src/stats/univariate/bootstrap.rs @@ -0,0 +1,161 @@ +#[cfg(test)] +macro_rules! test { + ($ty:ident) => { + mod $ty { + use approx::relative_eq; + use quickcheck::quickcheck; + use quickcheck::TestResult; + + use crate::stats::univariate::{Sample, mixed, self}; + + quickcheck!{ + fn mean(size: usize, start: usize, nresamples: usize) -> TestResult { + if let Some(v) = crate::stats::test::vec::<$ty>(size, start) { + let sample = Sample::new(&v[start..]); + + let means = if nresamples > 0 { + sample.bootstrap(nresamples, |s| (s.mean(),)).0 + } else { + return TestResult::discard(); + }; + + let min = sample.min(); + let max = sample.max(); + + TestResult::from_bool( + // Computed the correct number of resamples + means.len() == nresamples && + // No uninitialized values + means.iter().all(|&x| { + (x > min || relative_eq!(x, min)) && + (x < max || relative_eq!(x, max)) + }) + ) + } else { + TestResult::discard() + } + } + } + + quickcheck!{ + fn mean_median(size: usize, start: usize, nresamples: usize) -> TestResult { + if let Some(v) = crate::stats::test::vec::<$ty>(size, start) { + let sample = Sample::new(&v[start..]); + + let (means, medians) = if nresamples > 0 { + sample.bootstrap(nresamples, |s| (s.mean(), s.median())) + } else { + return TestResult::discard(); + }; + + let min = sample.min(); + let max = sample.max(); + + TestResult::from_bool( + // Computed the correct number of resamples + means.len() == nresamples && + medians.len() == nresamples && + // No uninitialized values + means.iter().all(|&x| { + (x > min || relative_eq!(x, min)) && + (x < max || relative_eq!(x, max)) + }) && + medians.iter().all(|&x| { + (x > min || relative_eq!(x, min)) && + (x < max || relative_eq!(x, max)) + }) + ) + } else { + TestResult::discard() + } + } + } + + quickcheck!{ + fn mixed_two_sample( + a_size: usize, a_start: usize, + b_size: usize, b_start: usize, + nresamples: usize + ) -> TestResult { + if let (Some(a), Some(b)) = + (crate::stats::test::vec::<$ty>(a_size, a_start), crate::stats::test::vec::<$ty>(b_size, b_start)) + { + let a = Sample::new(&a); + let b = Sample::new(&b); + + let distribution = if nresamples > 0 { + mixed::bootstrap(a, b, nresamples, |a, b| (a.mean() - b.mean(),)).0 + } else { + return TestResult::discard(); + }; + + let min = <$ty>::min(a.min() - b.max(), b.min() - a.max()); + let max = <$ty>::max(a.max() - b.min(), b.max() - a.min()); + + TestResult::from_bool( + // Computed the correct number of resamples + distribution.len() == nresamples && + // No uninitialized values + distribution.iter().all(|&x| { + (x > min || relative_eq!(x, min)) && + (x < max || relative_eq!(x, max)) + }) + ) + } else { + TestResult::discard() + } + } + } + + quickcheck!{ + fn two_sample( + a_size: usize, a_start: usize, + b_size: usize, b_start: usize, + nresamples: usize + ) -> TestResult { + if let (Some(a), Some(b)) = + (crate::stats::test::vec::<$ty>(a_size, a_start), crate::stats::test::vec::<$ty>(b_size, b_start)) + { + let a = Sample::new(&a[a_start..]); + let b = Sample::new(&b[b_start..]); + + let distribution = if nresamples > 0 { + univariate::bootstrap(a, b, nresamples, |a, b| (a.mean() - b.mean(),)).0 + } else { + return TestResult::discard(); + }; + + let min = <$ty>::min(a.min() - b.max(), b.min() - a.max()); + let max = <$ty>::max(a.max() - b.min(), b.max() - a.min()); + + // Computed the correct number of resamples + let pass = distribution.len() == nresamples && + // No uninitialized values + distribution.iter().all(|&x| { + (x > min || relative_eq!(x, min)) && + (x < max || relative_eq!(x, max)) + }); + + if !pass { + println!("A: {:?} (len={})", a.as_ref(), a.len()); + println!("B: {:?} (len={})", b.as_ref(), b.len()); + println!("Dist: {:?} (len={})", distribution.as_ref(), distribution.len()); + println!("Min: {}, Max: {}, nresamples: {}", + min, max, nresamples); + } + + TestResult::from_bool(pass) + } else { + TestResult::discard() + } + } + } + } + } +} + +#[cfg(test)] +mod test { + test!(f32); + test!(f64); +} diff --git a/src/stats/univariate/kde/kernel.rs b/src/stats/univariate/kde/kernel.rs new file mode 100755 index 0000000..b4204f5 --- /dev/null +++ b/src/stats/univariate/kde/kernel.rs @@ -0,0 +1,82 @@ +//! Kernels + +use crate::stats::float::Float; + +/// Kernel function +pub trait Kernel<A>: Copy + Sync +where + A: Float, +{ + /// Apply the kernel function to the given x-value. + fn evaluate(&self, x: A) -> A; +} + +/// Gaussian kernel +#[derive(Clone, Copy)] +pub struct Gaussian; + +impl<A> Kernel<A> for Gaussian +where + A: Float, +{ + fn evaluate(&self, x: A) -> A { + use std::f32::consts::PI; + + (x.powi(2).exp() * A::cast(2. * PI)).sqrt().recip() + } +} + +#[cfg(test)] +macro_rules! test { + ($ty:ident) => { + mod $ty { + mod gaussian { + use approx::relative_eq; + use quickcheck::quickcheck; + use quickcheck::TestResult; + + use crate::stats::univariate::kde::kernel::{Gaussian, Kernel}; + + quickcheck! { + fn symmetric(x: $ty) -> bool { + relative_eq!(Gaussian.evaluate(-x), Gaussian.evaluate(x)) + } + } + + // Any [a b] integral should be in the range [0 1] + quickcheck! { + fn integral(a: $ty, b: $ty) -> TestResult { + const DX: $ty = 1e-3; + + if a > b { + TestResult::discard() + } else { + let mut acc = 0.; + let mut x = a; + let mut y = Gaussian.evaluate(a); + + while x < b { + acc += DX * y / 2.; + + x += DX; + y = Gaussian.evaluate(x); + + acc += DX * y / 2.; + } + + TestResult::from_bool( + (acc > 0. || relative_eq!(acc, 0.)) && + (acc < 1. || relative_eq!(acc, 1.))) + } + } + } + } + } + }; +} + +#[cfg(test)] +mod test { + test!(f32); + test!(f64); +} diff --git a/src/stats/univariate/kde/mod.rs b/src/stats/univariate/kde/mod.rs new file mode 100755 index 0000000..efc27cd --- /dev/null +++ b/src/stats/univariate/kde/mod.rs @@ -0,0 +1,140 @@ +//! Kernel density estimation + +pub mod kernel; + +use self::kernel::Kernel; +use crate::stats::float::Float; +use crate::stats::univariate::Sample; +use rayon::prelude::*; + +/// Univariate kernel density estimator +pub struct Kde<'a, A, K> +where + A: Float, + K: Kernel<A>, +{ + bandwidth: A, + kernel: K, + sample: &'a Sample<A>, +} + +impl<'a, A, K> Kde<'a, A, K> +where + A: 'a + Float, + K: Kernel<A>, +{ + /// Creates a new kernel density estimator from the `sample`, using a kernel and estimating + /// the bandwidth using the method `bw` + pub fn new(sample: &'a Sample<A>, kernel: K, bw: Bandwidth) -> Kde<'a, A, K> { + Kde { + bandwidth: bw.estimate(sample), + kernel, + sample, + } + } + + /// Returns the bandwidth used by the estimator + pub fn bandwidth(&self) -> A { + self.bandwidth + } + + /// Maps the KDE over `xs` + /// + /// - Multihreaded + pub fn map(&self, xs: &[A]) -> Box<[A]> { + xs.par_iter() + .map(|&x| self.estimate(x)) + .collect::<Vec<_>>() + .into_boxed_slice() + } + + /// Estimates the probability density of `x` + pub fn estimate(&self, x: A) -> A { + let _0 = A::cast(0); + let slice = self.sample; + let h = self.bandwidth; + let n = A::cast(slice.len()); + + let sum = slice + .iter() + .fold(_0, |acc, &x_i| acc + self.kernel.evaluate((x - x_i) / h)); + + sum / (h * n) + } +} + +/// Method to estimate the bandwidth +pub enum Bandwidth { + /// Use Silverman's rule of thumb to estimate the bandwidth from the sample + Silverman, +} + +impl Bandwidth { + fn estimate<A: Float>(self, sample: &Sample<A>) -> A { + match self { + Bandwidth::Silverman => { + let factor = A::cast(4. / 3.); + let exponent = A::cast(1. / 5.); + let n = A::cast(sample.len()); + let sigma = sample.std_dev(None); + + sigma * (factor / n).powf(exponent) + } + } + } +} + +#[cfg(test)] +macro_rules! test { + ($ty:ident) => { + mod $ty { + use approx::relative_eq; + use quickcheck::quickcheck; + use quickcheck::TestResult; + + use crate::stats::univariate::kde::kernel::Gaussian; + use crate::stats::univariate::kde::{Bandwidth, Kde}; + use crate::stats::univariate::Sample; + + // The [-inf inf] integral of the estimated PDF should be one + quickcheck! { + fn integral(size: usize, start: usize) -> TestResult { + const DX: $ty = 1e-3; + + if let Some(v) = crate::stats::test::vec::<$ty>(size, start) { + let slice = &v[start..]; + let data = Sample::new(slice); + let kde = Kde::new(data, Gaussian, Bandwidth::Silverman); + let h = kde.bandwidth(); + // NB Obviously a [-inf inf] integral is not feasible, but this range works + // quite well + let (a, b) = (data.min() - 5. * h, data.max() + 5. * h); + + let mut acc = 0.; + let mut x = a; + let mut y = kde.estimate(a); + + while x < b { + acc += DX * y / 2.; + + x += DX; + y = kde.estimate(x); + + acc += DX * y / 2.; + } + + TestResult::from_bool(relative_eq!(acc, 1., epsilon = 2e-5)) + } else { + TestResult::discard() + } + } + } + } + }; +} + +#[cfg(test)] +mod test { + test!(f32); + test!(f64); +} diff --git a/src/stats/univariate/mixed.rs b/src/stats/univariate/mixed.rs new file mode 100755 index 0000000..5c0a59f --- /dev/null +++ b/src/stats/univariate/mixed.rs @@ -0,0 +1,57 @@ +//! Mixed bootstrap + +use crate::stats::float::Float; +use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; +use crate::stats::univariate::Resamples; +use crate::stats::univariate::Sample; +use rayon::prelude::*; + +/// Performs a *mixed* two-sample bootstrap +pub fn bootstrap<A, T, S>( + a: &Sample<A>, + b: &Sample<A>, + nresamples: usize, + statistic: S, +) -> T::Distributions +where + A: Float, + S: Fn(&Sample<A>, &Sample<A>) -> T + Sync, + T: Tuple + Send, + T::Distributions: Send, + T::Builder: Send, +{ + let n_a = a.len(); + let n_b = b.len(); + let mut c = Vec::with_capacity(n_a + n_b); + c.extend_from_slice(a); + c.extend_from_slice(b); + let c = Sample::new(&c); + + (0..nresamples) + .into_par_iter() + .map_init( + || Resamples::new(c), + |resamples, _| { + let resample = resamples.next(); + let a: &Sample<A> = Sample::new(&resample[..n_a]); + let b: &Sample<A> = Sample::new(&resample[n_a..]); + + statistic(a, b) + }, + ) + .fold( + || T::Builder::new(0), + |mut sub_distributions, sample| { + sub_distributions.push(sample); + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() +} diff --git a/src/stats/univariate/mod.rs b/src/stats/univariate/mod.rs new file mode 100755 index 0000000..8dfb5f8 --- /dev/null +++ b/src/stats/univariate/mod.rs @@ -0,0 +1,72 @@ +//! Univariate analysis + +mod bootstrap; +mod percentiles; +mod resamples; +mod sample; + +pub mod kde; +pub mod mixed; +pub mod outliers; + +use crate::stats::float::Float; +use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; +use rayon::prelude::*; +use std::cmp; + +use self::resamples::Resamples; + +pub use self::percentiles::Percentiles; +pub use self::sample::Sample; + +/// Performs a two-sample bootstrap +/// +/// - Multithreaded +/// - Time: `O(nresamples)` +/// - Memory: `O(nresamples)` +#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_lossless))] +pub fn bootstrap<A, B, T, S>( + a: &Sample<A>, + b: &Sample<B>, + nresamples: usize, + statistic: S, +) -> T::Distributions +where + A: Float, + B: Float, + S: Fn(&Sample<A>, &Sample<B>) -> T + Sync, + T: Tuple + Send, + T::Distributions: Send, + T::Builder: Send, +{ + let nresamples_sqrt = (nresamples as f64).sqrt().ceil() as usize; + let per_chunk = (nresamples + nresamples_sqrt - 1) / nresamples_sqrt; + + (0..nresamples_sqrt) + .into_par_iter() + .map_init( + || (Resamples::new(a), Resamples::new(b)), + |(a_resamples, b_resamples), i| { + let start = i * per_chunk; + let end = cmp::min((i + 1) * per_chunk, nresamples); + let a_resample = a_resamples.next(); + + let mut sub_distributions: T::Builder = + TupledDistributionsBuilder::new(end - start); + + for _ in start..end { + let b_resample = b_resamples.next(); + sub_distributions.push(statistic(a_resample, b_resample)); + } + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() +} diff --git a/src/stats/univariate/outliers/mod.rs b/src/stats/univariate/outliers/mod.rs new file mode 100755 index 0000000..b8ed7c7 --- /dev/null +++ b/src/stats/univariate/outliers/mod.rs @@ -0,0 +1,7 @@ +//! Classification of outliers +//! +//! WARNING: There's no formal/mathematical definition of what an outlier actually is. Therefore, +//! all outlier classifiers are *subjective*, however some classifiers that have become *de facto* +//! standard are provided here. + +pub mod tukey; diff --git a/src/stats/univariate/outliers/tukey.rs b/src/stats/univariate/outliers/tukey.rs new file mode 100755 index 0000000..bfd08f1 --- /dev/null +++ b/src/stats/univariate/outliers/tukey.rs @@ -0,0 +1,291 @@ +//! Tukey's method +//! +//! The original method uses two "fences" to classify the data. All the observations "inside" the +//! fences are considered "normal", and the rest are considered outliers. +//! +//! The fences are computed from the quartiles of the sample, according to the following formula: +//! +//! ``` ignore +//! // q1, q3 are the first and third quartiles +//! let iqr = q3 - q1; // The interquartile range +//! let (f1, f2) = (q1 - 1.5 * iqr, q3 + 1.5 * iqr); // the "fences" +//! +//! let is_outlier = |x| if x > f1 && x < f2 { true } else { false }; +//! ``` +//! +//! The classifier provided here adds two extra outer fences: +//! +//! ``` ignore +//! let (f3, f4) = (q1 - 3 * iqr, q3 + 3 * iqr); // the outer "fences" +//! ``` +//! +//! The extra fences add a sense of "severity" to the classification. Data points outside of the +//! outer fences are considered "severe" outliers, whereas points outside the inner fences are just +//! "mild" outliers, and, as the original method, everything inside the inner fences is considered +//! "normal" data. +//! +//! Some ASCII art for the visually oriented people: +//! +//! ``` ignore +//! LOW-ish NORMAL-ish HIGH-ish +//! x | + | o o o o o o o | + | x +//! f3 f1 f2 f4 +//! +//! Legend: +//! o: "normal" data (not an outlier) +//! +: "mild" outlier +//! x: "severe" outlier +//! ``` + +use std::iter::IntoIterator; +use std::ops::{Deref, Index}; +use std::slice; + +use crate::stats::float::Float; +use crate::stats::univariate::Sample; + +use self::Label::*; + +/// A classified/labeled sample. +/// +/// The labeled data can be accessed using the indexing operator. The order of the data points is +/// retained. +/// +/// NOTE: Due to limitations in the indexing traits, only the label is returned. Once the +/// `IndexGet` trait lands in stdlib, the indexing operation will return a `(data_point, label)` +/// pair. +#[derive(Clone, Copy)] +pub struct LabeledSample<'a, A> +where + A: Float, +{ + fences: (A, A, A, A), + sample: &'a Sample<A>, +} + +impl<'a, A> LabeledSample<'a, A> +where + A: Float, +{ + /// Returns the number of data points per label + /// + /// - Time: `O(length)` + #[cfg_attr(feature = "cargo-clippy", allow(clippy::similar_names))] + pub fn count(&self) -> (usize, usize, usize, usize, usize) { + let (mut los, mut lom, mut noa, mut him, mut his) = (0, 0, 0, 0, 0); + + for (_, label) in self { + match label { + LowSevere => { + los += 1; + } + LowMild => { + lom += 1; + } + NotAnOutlier => { + noa += 1; + } + HighMild => { + him += 1; + } + HighSevere => { + his += 1; + } + } + } + + (los, lom, noa, him, his) + } + + /// Returns the fences used to classify the outliers + pub fn fences(&self) -> (A, A, A, A) { + self.fences + } + + /// Returns an iterator over the labeled data + pub fn iter(&self) -> Iter<'a, A> { + Iter { + fences: self.fences, + iter: self.sample.iter(), + } + } +} + +impl<'a, A> Deref for LabeledSample<'a, A> +where + A: Float, +{ + type Target = Sample<A>; + + fn deref(&self) -> &Sample<A> { + self.sample + } +} + +// FIXME Use the `IndexGet` trait +impl<'a, A> Index<usize> for LabeledSample<'a, A> +where + A: Float, +{ + type Output = Label; + + #[cfg_attr(feature = "cargo-clippy", allow(clippy::similar_names))] + fn index(&self, i: usize) -> &Label { + static LOW_SEVERE: Label = LowSevere; + static LOW_MILD: Label = LowMild; + static HIGH_MILD: Label = HighMild; + static HIGH_SEVERE: Label = HighSevere; + static NOT_AN_OUTLIER: Label = NotAnOutlier; + + let x = self.sample[i]; + let (lost, lomt, himt, hist) = self.fences; + + if x < lost { + &LOW_SEVERE + } else if x > hist { + &HIGH_SEVERE + } else if x < lomt { + &LOW_MILD + } else if x > himt { + &HIGH_MILD + } else { + &NOT_AN_OUTLIER + } + } +} + +impl<'a, 'b, A> IntoIterator for &'b LabeledSample<'a, A> +where + A: Float, +{ + type Item = (A, Label); + type IntoIter = Iter<'a, A>; + + fn into_iter(self) -> Iter<'a, A> { + self.iter() + } +} + +/// Iterator over the labeled data +pub struct Iter<'a, A> +where + A: Float, +{ + fences: (A, A, A, A), + iter: slice::Iter<'a, A>, +} + +impl<'a, A> Iterator for Iter<'a, A> +where + A: Float, +{ + type Item = (A, Label); + + #[cfg_attr(feature = "cargo-clippy", allow(clippy::similar_names))] + fn next(&mut self) -> Option<(A, Label)> { + self.iter.next().map(|&x| { + let (lost, lomt, himt, hist) = self.fences; + + let label = if x < lost { + LowSevere + } else if x > hist { + HighSevere + } else if x < lomt { + LowMild + } else if x > himt { + HighMild + } else { + NotAnOutlier + }; + + (x, label) + }) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.iter.size_hint() + } +} + +/// Labels used to classify outliers +pub enum Label { + /// A "mild" outlier in the "high" spectrum + HighMild, + /// A "severe" outlier in the "high" spectrum + HighSevere, + /// A "mild" outlier in the "low" spectrum + LowMild, + /// A "severe" outlier in the "low" spectrum + LowSevere, + /// A normal data point + NotAnOutlier, +} + +impl Label { + /// Checks if the data point has an "unusually" high value + pub fn is_high(&self) -> bool { + match *self { + HighMild | HighSevere => true, + _ => false, + } + } + + /// Checks if the data point is labeled as a "mild" outlier + pub fn is_mild(&self) -> bool { + match *self { + HighMild | LowMild => true, + _ => false, + } + } + + /// Checks if the data point has an "unusually" low value + pub fn is_low(&self) -> bool { + match *self { + LowMild | LowSevere => true, + _ => false, + } + } + + /// Checks if the data point is labeled as an outlier + pub fn is_outlier(&self) -> bool { + match *self { + NotAnOutlier => false, + _ => true, + } + } + + /// Checks if the data point is labeled as a "severe" outlier + pub fn is_severe(&self) -> bool { + match *self { + HighSevere | LowSevere => true, + _ => false, + } + } +} + +/// Classifies the sample, and returns a labeled sample. +/// +/// - Time: `O(N log N) where N = length` +pub fn classify<A>(sample: &Sample<A>) -> LabeledSample<'_, A> +where + A: Float, + usize: cast::From<A, Output = Result<usize, cast::Error>>, +{ + let (q1, _, q3) = sample.percentiles().quartiles(); + let iqr = q3 - q1; + + // Mild + let k_m = A::cast(1.5_f32); + // Severe + let k_s = A::cast(3); + + LabeledSample { + fences: ( + q1 - k_s * iqr, + q1 - k_m * iqr, + q3 + k_m * iqr, + q3 + k_s * iqr, + ), + sample, + } +} diff --git a/src/stats/univariate/percentiles.rs b/src/stats/univariate/percentiles.rs new file mode 100755 index 0000000..be6bcf3 --- /dev/null +++ b/src/stats/univariate/percentiles.rs @@ -0,0 +1,80 @@ +use crate::stats::float::Float; +use cast::{self, usize}; + +/// A "view" into the percentiles of a sample +pub struct Percentiles<A>(Box<[A]>) +where + A: Float; + +// TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module +impl<A> Percentiles<A> +where + A: Float, + usize: cast::From<A, Output = Result<usize, cast::Error>>, +{ + /// Returns the percentile at `p`% + /// + /// Safety: + /// + /// - Make sure that `p` is in the range `[0, 100]` + unsafe fn at_unchecked(&self, p: A) -> A { + let _100 = A::cast(100); + debug_assert!(p >= A::cast(0) && p <= _100); + debug_assert!(self.0.len() > 0); + let len = self.0.len() - 1; + + if p == _100 { + self.0[len] + } else { + let rank = (p / _100) * A::cast(len); + let integer = rank.floor(); + let fraction = rank - integer; + let n = usize(integer).unwrap(); + let &floor = self.0.get_unchecked(n); + let &ceiling = self.0.get_unchecked(n + 1); + + floor + (ceiling - floor) * fraction + } + } + + /// Returns the percentile at `p`% + /// + /// # Panics + /// + /// Panics if `p` is outside the closed `[0, 100]` range + pub fn at(&self, p: A) -> A { + let _0 = A::cast(0); + let _100 = A::cast(100); + + assert!(p >= _0 && p <= _100); + assert!(self.0.len() > 0); + + unsafe { self.at_unchecked(p) } + } + + /// Returns the interquartile range + pub fn iqr(&self) -> A { + unsafe { + let q1 = self.at_unchecked(A::cast(25)); + let q3 = self.at_unchecked(A::cast(75)); + + q3 - q1 + } + } + + /// Returns the 50th percentile + pub fn median(&self) -> A { + unsafe { self.at_unchecked(A::cast(50)) } + } + + /// Returns the 25th, 50th and 75th percentiles + pub fn quartiles(&self) -> (A, A, A) { + unsafe { + ( + self.at_unchecked(A::cast(25)), + self.at_unchecked(A::cast(50)), + self.at_unchecked(A::cast(75)), + ) + } + } +} diff --git a/src/stats/univariate/resamples.rs b/src/stats/univariate/resamples.rs new file mode 100755 index 0000000..831bc7a --- /dev/null +++ b/src/stats/univariate/resamples.rs @@ -0,0 +1,117 @@ +use std::mem; + +use crate::stats::float::Float; +use crate::stats::rand_util::{new_rng, Rng}; +use crate::stats::univariate::Sample; + +pub struct Resamples<'a, A> +where + A: Float, +{ + rng: Rng, + sample: &'a [A], + stage: Option<Vec<A>>, +} + +#[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))] +impl<'a, A> Resamples<'a, A> +where + A: 'a + Float, +{ + pub fn new(sample: &'a Sample<A>) -> Resamples<'a, A> { + let slice = sample; + + Resamples { + rng: new_rng(), + sample: slice, + stage: None, + } + } + + pub fn next(&mut self) -> &Sample<A> { + let n = self.sample.len(); + let rng = &mut self.rng; + + match self.stage { + None => { + let mut stage = Vec::with_capacity(n); + + for _ in 0..n { + let idx = rng.rand_range(0u64..(self.sample.len() as u64)); + stage.push(self.sample[idx as usize]) + } + + self.stage = Some(stage); + } + Some(ref mut stage) => { + for elem in stage.iter_mut() { + let idx = rng.rand_range(0u64..(self.sample.len() as u64)); + *elem = self.sample[idx as usize] + } + } + } + + if let Some(ref v) = self.stage { + unsafe { mem::transmute::<&[_], _>(v) } + } else { + unreachable!(); + } + } +} + +#[cfg(test)] +mod test { + use quickcheck::quickcheck; + use quickcheck::TestResult; + use std::collections::HashSet; + + use crate::stats::univariate::resamples::Resamples; + use crate::stats::univariate::Sample; + + // Check that the resample is a subset of the sample + quickcheck! { + fn subset(size: usize, nresamples: usize) -> TestResult { + if size > 1 { + let v: Vec<_> = (0..size).map(|i| i as f32).collect(); + let sample = Sample::new(&v); + let mut resamples = Resamples::new(sample); + let sample = v.iter().map(|&x| x as i64).collect::<HashSet<_>>(); + + TestResult::from_bool((0..nresamples).all(|_| { + let resample = resamples.next() + + .iter() + .map(|&x| x as i64) + .collect::<HashSet<_>>(); + + resample.is_subset(&sample) + })) + } else { + TestResult::discard() + } + } + } + + #[test] + fn different_subsets() { + let size = 1000; + let v: Vec<_> = (0..size).map(|i| i as f32).collect(); + let sample = Sample::new(&v); + let mut resamples = Resamples::new(sample); + + // Hypothetically, we might see one duplicate, but more than one is likely to be a bug. + let mut num_duplicated = 0; + for _ in 0..1000 { + let sample_1 = resamples.next().iter().cloned().collect::<Vec<_>>(); + let sample_2 = resamples.next().iter().cloned().collect::<Vec<_>>(); + + if sample_1 == sample_2 { + num_duplicated += 1; + } + } + + if num_duplicated > 1 { + panic!("Found {} duplicate samples", num_duplicated); + } + } +} diff --git a/src/stats/univariate/sample.rs b/src/stats/univariate/sample.rs new file mode 100755 index 0000000..8f10db7 --- /dev/null +++ b/src/stats/univariate/sample.rs @@ -0,0 +1,255 @@ +use std::{mem, ops}; + +use crate::stats::float::Float; +use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; +use crate::stats::univariate::Percentiles; +use crate::stats::univariate::Resamples; +use rayon::prelude::*; + +/// A collection of data points drawn from a population +/// +/// Invariants: +/// +/// - The sample contains at least 2 data points +/// - The sample contains no `NaN`s +pub struct Sample<A>([A]); + +// TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module +impl<A> Sample<A> +where + A: Float, +{ + /// Creates a new sample from an existing slice + /// + /// # Panics + /// + /// Panics if `slice` contains any `NaN` or if `slice` has less than two elements + #[cfg_attr(feature = "cargo-clippy", allow(clippy::new_ret_no_self))] + pub fn new(slice: &[A]) -> &Sample<A> { + assert!(slice.len() > 1 && slice.iter().all(|x| !x.is_nan())); + + unsafe { mem::transmute(slice) } + } + + /// Returns the biggest element in the sample + /// + /// - Time: `O(length)` + pub fn max(&self) -> A { + let mut elems = self.iter(); + + match elems.next() { + Some(&head) => elems.fold(head, |a, &b| a.max(b)), + // NB `unreachable!` because `Sample` is guaranteed to have at least one data point + None => unreachable!(), + } + } + + /// Returns the arithmetic average of the sample + /// + /// - Time: `O(length)` + pub fn mean(&self) -> A { + let n = self.len(); + + self.sum() / A::cast(n) + } + + /// Returns the median absolute deviation + /// + /// The `median` can be optionally passed along to speed up (2X) the computation + /// + /// - Time: `O(length)` + /// - Memory: `O(length)` + pub fn median_abs_dev(&self, median: Option<A>) -> A + where + usize: cast::From<A, Output = Result<usize, cast::Error>>, + { + let median = median.unwrap_or_else(|| self.percentiles().median()); + + // NB Although this operation can be SIMD accelerated, the gain is negligible because the + // bottle neck is the sorting operation which is part of the computation of the median + let abs_devs = self.iter().map(|&x| (x - median).abs()).collect::<Vec<_>>(); + + let abs_devs: &Self = Self::new(&abs_devs); + + abs_devs.percentiles().median() * A::cast(1.4826) + } + + /// Returns the median absolute deviation as a percentage of the median + /// + /// - Time: `O(length)` + /// - Memory: `O(length)` + pub fn median_abs_dev_pct(&self) -> A + where + usize: cast::From<A, Output = Result<usize, cast::Error>>, + { + let _100 = A::cast(100); + let median = self.percentiles().median(); + let mad = self.median_abs_dev(Some(median)); + + (mad / median) * _100 + } + + /// Returns the smallest element in the sample + /// + /// - Time: `O(length)` + pub fn min(&self) -> A { + let mut elems = self.iter(); + + match elems.next() { + Some(&elem) => elems.fold(elem, |a, &b| a.min(b)), + // NB `unreachable!` because `Sample` is guaranteed to have at least one data point + None => unreachable!(), + } + } + + /// Returns a "view" into the percentiles of the sample + /// + /// This "view" makes consecutive computations of percentiles much faster (`O(1)`) + /// + /// - Time: `O(N log N) where N = length` + /// - Memory: `O(length)` + pub fn percentiles(&self) -> Percentiles<A> + where + usize: cast::From<A, Output = Result<usize, cast::Error>>, + { + use std::cmp::Ordering; + + // NB This function assumes that there are no `NaN`s in the sample + fn cmp<T>(a: &T, b: &T) -> Ordering + where + T: PartialOrd, + { + match a.partial_cmp(b) { + Some(o) => o, + // Arbitrary way to handle NaNs that should never happen + None => Ordering::Equal, + } + } + + let mut v = self.to_vec().into_boxed_slice(); + v.par_sort_unstable_by(cmp); + + // NB :-1: to intra-crate privacy rules + unsafe { mem::transmute(v) } + } + + /// Returns the standard deviation of the sample + /// + /// The `mean` can be optionally passed along to speed up (2X) the computation + /// + /// - Time: `O(length)` + pub fn std_dev(&self, mean: Option<A>) -> A { + self.var(mean).sqrt() + } + + /// Returns the standard deviation as a percentage of the mean + /// + /// - Time: `O(length)` + pub fn std_dev_pct(&self) -> A { + let _100 = A::cast(100); + let mean = self.mean(); + let std_dev = self.std_dev(Some(mean)); + + (std_dev / mean) * _100 + } + + /// Returns the sum of all the elements of the sample + /// + /// - Time: `O(length)` + pub fn sum(&self) -> A { + crate::stats::sum(self) + } + + /// Returns the t score between these two samples + /// + /// - Time: `O(length)` + pub fn t(&self, other: &Sample<A>) -> A { + let (x_bar, y_bar) = (self.mean(), other.mean()); + let (s2_x, s2_y) = (self.var(Some(x_bar)), other.var(Some(y_bar))); + let n_x = A::cast(self.len()); + let n_y = A::cast(other.len()); + let num = x_bar - y_bar; + let den = (s2_x / n_x + s2_y / n_y).sqrt(); + + num / den + } + + /// Returns the variance of the sample + /// + /// The `mean` can be optionally passed along to speed up (2X) the computation + /// + /// - Time: `O(length)` + pub fn var(&self, mean: Option<A>) -> A { + use std::ops::Add; + + let mean = mean.unwrap_or_else(|| self.mean()); + let slice = self; + + let sum = slice + .iter() + .map(|&x| (x - mean).powi(2)) + .fold(A::cast(0), Add::add); + + sum / A::cast(slice.len() - 1) + } + + // TODO Remove the `T` parameter in favor of `S::Output` + /// Returns the bootstrap distributions of the parameters estimated by the 1-sample statistic + /// + /// - Multi-threaded + /// - Time: `O(nresamples)` + /// - Memory: `O(nresamples)` + pub fn bootstrap<T, S>(&self, nresamples: usize, statistic: S) -> T::Distributions + where + S: Fn(&Sample<A>) -> T + Sync, + T: Tuple + Send, + T::Distributions: Send, + T::Builder: Send, + { + (0..nresamples) + .into_par_iter() + .map_init( + || Resamples::new(self), + |resamples, _| statistic(resamples.next()), + ) + .fold( + || T::Builder::new(0), + |mut sub_distributions, sample| { + sub_distributions.push(sample); + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() + } + + #[cfg(test)] + pub fn iqr(&self) -> A + where + usize: cast::From<A, Output = Result<usize, cast::Error>>, + { + self.percentiles().iqr() + } + + #[cfg(test)] + pub fn median(&self) -> A + where + usize: cast::From<A, Output = Result<usize, cast::Error>>, + { + self.percentiles().median() + } +} + +impl<A> ops::Deref for Sample<A> { + type Target = [A]; + + fn deref(&self) -> &[A] { + &self.0 + } +} |