aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorJakub Kotur <qtr@google.com>2021-03-16 20:53:15 +0000
committerAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>2021-03-16 20:53:15 +0000
commita55d029ebdfba8e158381f1e12cf6a1c689cf980 (patch)
tree8ab32923fe4a5e70e694e6147ea7f6783bc1b04b /src
parent71f53f93f889decb4b2faafb4eaea7b0e9c70722 (diff)
parent875984d49b275881c241a65b23e863836499bf4e (diff)
downloadcriterion-a55d029ebdfba8e158381f1e12cf6a1c689cf980.tar.gz
Initial import of criterion-0.3.3. am: 704f579139 am: 3aaf8f3b8b am: afadec1622 am: 875984d49b
Original change: https://android-review.googlesource.com/c/platform/external/rust/crates/criterion/+/1621184 Change-Id: I7e2886be4aa37403b1bccb95ee32656526f04442
Diffstat (limited to 'src')
-rwxr-xr-xsrc/analysis/compare.rs143
-rwxr-xr-xsrc/analysis/mod.rs358
-rwxr-xr-xsrc/benchmark.rs611
-rwxr-xr-xsrc/benchmark_group.rs497
-rwxr-xr-xsrc/connection.rs370
-rwxr-xr-xsrc/csv_report.rs90
-rwxr-xr-xsrc/error.rs72
-rwxr-xr-xsrc/estimate.rs184
-rwxr-xr-xsrc/format.rs104
-rwxr-xr-xsrc/fs.rs112
-rwxr-xr-xsrc/html/benchmark_report.html.tt308
-rwxr-xr-xsrc/html/index.html.tt119
-rwxr-xr-xsrc/html/mod.rs837
-rwxr-xr-xsrc/html/report_link.html.tt5
-rwxr-xr-xsrc/html/summary_report.html.tt109
-rwxr-xr-xsrc/kde.rs41
-rwxr-xr-xsrc/lib.rs1899
-rwxr-xr-xsrc/macros.rs132
-rwxr-xr-xsrc/macros_private.rs47
-rwxr-xr-xsrc/measurement.rs212
-rwxr-xr-xsrc/plot/gnuplot_backend/distributions.rs310
-rwxr-xr-xsrc/plot/gnuplot_backend/iteration_times.rs173
-rwxr-xr-xsrc/plot/gnuplot_backend/mod.rs254
-rwxr-xr-xsrc/plot/gnuplot_backend/pdf.rs392
-rwxr-xr-xsrc/plot/gnuplot_backend/regression.rs279
-rwxr-xr-xsrc/plot/gnuplot_backend/summary.rs211
-rwxr-xr-xsrc/plot/gnuplot_backend/t_test.rs65
-rwxr-xr-xsrc/plot/mod.rs103
-rwxr-xr-xsrc/plot/plotters_backend/distributions.rs309
-rwxr-xr-xsrc/plot/plotters_backend/iteration_times.rs138
-rwxr-xr-xsrc/plot/plotters_backend/mod.rs232
-rwxr-xr-xsrc/plot/plotters_backend/pdf.rs307
-rwxr-xr-xsrc/plot/plotters_backend/regression.rs234
-rwxr-xr-xsrc/plot/plotters_backend/summary.rs256
-rwxr-xr-xsrc/plot/plotters_backend/t_test.rs59
-rwxr-xr-xsrc/profiler.rs28
-rwxr-xr-xsrc/report.rs880
-rwxr-xr-xsrc/routine.rs240
-rwxr-xr-xsrc/stats/bivariate/bootstrap.rs83
-rwxr-xr-xsrc/stats/bivariate/mod.rs131
-rwxr-xr-xsrc/stats/bivariate/regression.rs53
-rwxr-xr-xsrc/stats/bivariate/resamples.rs61
-rwxr-xr-xsrc/stats/float.rs15
-rwxr-xr-xsrc/stats/mod.rs112
-rwxr-xr-xsrc/stats/rand_util.rs21
-rwxr-xr-xsrc/stats/test.rs16
-rwxr-xr-xsrc/stats/tuple.rs253
-rwxr-xr-xsrc/stats/univariate/bootstrap.rs161
-rwxr-xr-xsrc/stats/univariate/kde/kernel.rs82
-rwxr-xr-xsrc/stats/univariate/kde/mod.rs140
-rwxr-xr-xsrc/stats/univariate/mixed.rs57
-rwxr-xr-xsrc/stats/univariate/mod.rs72
-rwxr-xr-xsrc/stats/univariate/outliers/mod.rs7
-rwxr-xr-xsrc/stats/univariate/outliers/tukey.rs291
-rwxr-xr-xsrc/stats/univariate/percentiles.rs80
-rwxr-xr-xsrc/stats/univariate/resamples.rs117
-rwxr-xr-xsrc/stats/univariate/sample.rs255
57 files changed, 12727 insertions, 0 deletions
diff --git a/src/analysis/compare.rs b/src/analysis/compare.rs
new file mode 100755
index 0000000..a49407d
--- /dev/null
+++ b/src/analysis/compare.rs
@@ -0,0 +1,143 @@
+use crate::stats::univariate::Sample;
+use crate::stats::univariate::{self, mixed};
+use crate::stats::Distribution;
+
+use crate::benchmark::BenchmarkConfig;
+use crate::error::Result;
+use crate::estimate::{
+ build_change_estimates, ChangeDistributions, ChangeEstimates, ChangePointEstimates, Estimates,
+};
+use crate::measurement::Measurement;
+use crate::report::BenchmarkId;
+use crate::{fs, Criterion, SavedSample};
+
+// Common comparison procedure
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::type_complexity))]
+pub(crate) fn common<M: Measurement>(
+ id: &BenchmarkId,
+ avg_times: &Sample<f64>,
+ config: &BenchmarkConfig,
+ criterion: &Criterion<M>,
+) -> Result<(
+ f64,
+ Distribution<f64>,
+ ChangeEstimates,
+ ChangeDistributions,
+ Vec<f64>,
+ Vec<f64>,
+ Vec<f64>,
+ Estimates,
+)> {
+ let mut sample_file = criterion.output_directory.clone();
+ sample_file.push(id.as_directory_name());
+ sample_file.push(&criterion.baseline_directory);
+ sample_file.push("sample.json");
+ let sample: SavedSample = fs::load(&sample_file)?;
+ let SavedSample { iters, times, .. } = sample;
+
+ let mut estimates_file = criterion.output_directory.clone();
+ estimates_file.push(id.as_directory_name());
+ estimates_file.push(&criterion.baseline_directory);
+ estimates_file.push("estimates.json");
+ let base_estimates: Estimates = fs::load(&estimates_file)?;
+
+ let base_avg_times: Vec<f64> = iters
+ .iter()
+ .zip(times.iter())
+ .map(|(iters, elapsed)| elapsed / iters)
+ .collect();
+ let base_avg_time_sample = Sample::new(&base_avg_times);
+
+ let mut change_dir = criterion.output_directory.clone();
+ change_dir.push(id.as_directory_name());
+ change_dir.push("change");
+ fs::mkdirp(&change_dir)?;
+ let (t_statistic, t_distribution) = t_test(avg_times, base_avg_time_sample, config);
+
+ let (estimates, relative_distributions) =
+ estimates(id, avg_times, base_avg_time_sample, config, criterion);
+ Ok((
+ t_statistic,
+ t_distribution,
+ estimates,
+ relative_distributions,
+ iters,
+ times,
+ base_avg_times.clone(),
+ base_estimates,
+ ))
+}
+
+// Performs a two sample t-test
+fn t_test(
+ avg_times: &Sample<f64>,
+ base_avg_times: &Sample<f64>,
+ config: &BenchmarkConfig,
+) -> (f64, Distribution<f64>) {
+ let nresamples = config.nresamples;
+
+ let t_statistic = avg_times.t(base_avg_times);
+ let t_distribution = elapsed!(
+ "Bootstrapping the T distribution",
+ mixed::bootstrap(avg_times, base_avg_times, nresamples, |a, b| (a.t(b),))
+ )
+ .0;
+
+ // HACK: Filter out non-finite numbers, which can happen sometimes when sample size is very small.
+ // Downstream code doesn't like non-finite values here.
+ let t_distribution = Distribution::from(
+ t_distribution
+ .iter()
+ .filter(|a| a.is_finite())
+ .cloned()
+ .collect::<Vec<_>>()
+ .into_boxed_slice(),
+ );
+
+ (t_statistic, t_distribution)
+}
+
+// Estimates the relative change in the statistics of the population
+fn estimates<M: Measurement>(
+ id: &BenchmarkId,
+ avg_times: &Sample<f64>,
+ base_avg_times: &Sample<f64>,
+ config: &BenchmarkConfig,
+ criterion: &Criterion<M>,
+) -> (ChangeEstimates, ChangeDistributions) {
+ fn stats(a: &Sample<f64>, b: &Sample<f64>) -> (f64, f64) {
+ (
+ a.mean() / b.mean() - 1.,
+ a.percentiles().median() / b.percentiles().median() - 1.,
+ )
+ }
+
+ let cl = config.confidence_level;
+ let nresamples = config.nresamples;
+
+ let (dist_mean, dist_median) = elapsed!(
+ "Bootstrapping the relative statistics",
+ univariate::bootstrap(avg_times, base_avg_times, nresamples, stats)
+ );
+
+ let distributions = ChangeDistributions {
+ mean: dist_mean,
+ median: dist_median,
+ };
+
+ let (mean, median) = stats(avg_times, base_avg_times);
+ let points = ChangePointEstimates { mean, median };
+
+ let estimates = build_change_estimates(&distributions, &points, cl);
+
+ {
+ log_if_err!({
+ let mut estimates_path = criterion.output_directory.clone();
+ estimates_path.push(id.as_directory_name());
+ estimates_path.push("change");
+ estimates_path.push("estimates.json");
+ fs::save(&estimates, &estimates_path)
+ });
+ }
+ (estimates, distributions)
+}
diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs
new file mode 100755
index 0000000..caa948d
--- /dev/null
+++ b/src/analysis/mod.rs
@@ -0,0 +1,358 @@
+use std::path::Path;
+
+use crate::stats::bivariate::regression::Slope;
+use crate::stats::bivariate::Data;
+use crate::stats::univariate::outliers::tukey;
+use crate::stats::univariate::Sample;
+use crate::stats::{Distribution, Tails};
+
+use crate::benchmark::BenchmarkConfig;
+use crate::connection::OutgoingMessage;
+use crate::estimate::{
+ build_estimates, ConfidenceInterval, Distributions, Estimate, Estimates, PointEstimates,
+};
+use crate::fs;
+use crate::measurement::Measurement;
+use crate::report::{BenchmarkId, ReportContext};
+use crate::routine::Routine;
+use crate::{Baseline, Criterion, SavedSample, Throughput};
+
+macro_rules! elapsed {
+ ($msg:expr, $block:expr) => {{
+ let start = ::std::time::Instant::now();
+ let out = $block;
+ let elapsed = &start.elapsed();
+
+ info!(
+ "{} took {}",
+ $msg,
+ crate::format::time(crate::DurationExt::to_nanos(elapsed) as f64)
+ );
+
+ out
+ }};
+}
+
+mod compare;
+
+// Common analysis procedure
+pub(crate) fn common<M: Measurement, T: ?Sized>(
+ id: &BenchmarkId,
+ routine: &mut dyn Routine<M, T>,
+ config: &BenchmarkConfig,
+ criterion: &Criterion<M>,
+ report_context: &ReportContext,
+ parameter: &T,
+ throughput: Option<Throughput>,
+) {
+ criterion.report.benchmark_start(id, report_context);
+
+ if let Baseline::Compare = criterion.baseline {
+ if !base_dir_exists(
+ id,
+ &criterion.baseline_directory,
+ &criterion.output_directory,
+ ) {
+ panic!(format!(
+ "Baseline '{base}' must exist before comparison is allowed; try --save-baseline {base}",
+ base=criterion.baseline_directory,
+ ));
+ }
+ }
+
+ let (sampling_mode, iters, times);
+ if let Some(baseline) = &criterion.load_baseline {
+ let mut sample_path = criterion.output_directory.clone();
+ sample_path.push(id.as_directory_name());
+ sample_path.push(baseline);
+ sample_path.push("sample.json");
+ let loaded = fs::load::<SavedSample, _>(&sample_path);
+
+ match loaded {
+ Err(err) => panic!(
+ "Baseline '{base}' must exist before it can be loaded; try --save-baseline {base}. Error: {err}",
+ base = baseline, err = err
+ ),
+ Ok(samples) => {
+ sampling_mode = samples.sampling_mode;
+ iters = samples.iters.into_boxed_slice();
+ times = samples.times.into_boxed_slice();
+ }
+ }
+ } else {
+ let sample = routine.sample(
+ &criterion.measurement,
+ id,
+ config,
+ criterion,
+ report_context,
+ parameter,
+ );
+ sampling_mode = sample.0;
+ iters = sample.1;
+ times = sample.2;
+
+ if let Some(conn) = &criterion.connection {
+ conn.send(&OutgoingMessage::MeasurementComplete {
+ id: id.into(),
+ iters: &iters,
+ times: &times,
+ plot_config: (&report_context.plot_config).into(),
+ sampling_method: sampling_mode.into(),
+ benchmark_config: config.into(),
+ })
+ .unwrap();
+
+ conn.serve_value_formatter(criterion.measurement.formatter())
+ .unwrap();
+ }
+ }
+
+ criterion.report.analysis(id, report_context);
+
+ let avg_times = iters
+ .iter()
+ .zip(times.iter())
+ .map(|(&iters, &elapsed)| elapsed / iters)
+ .collect::<Vec<f64>>();
+ let avg_times = Sample::new(&avg_times);
+
+ if criterion.connection.is_none() && criterion.load_baseline.is_none() {
+ log_if_err!({
+ let mut new_dir = criterion.output_directory.clone();
+ new_dir.push(id.as_directory_name());
+ new_dir.push("new");
+ fs::mkdirp(&new_dir)
+ });
+ }
+
+ let data = Data::new(&iters, &times);
+ let labeled_sample = tukey::classify(avg_times);
+ if criterion.connection.is_none() {
+ log_if_err!({
+ let mut tukey_file = criterion.output_directory.to_owned();
+ tukey_file.push(id.as_directory_name());
+ tukey_file.push("new");
+ tukey_file.push("tukey.json");
+ fs::save(&labeled_sample.fences(), &tukey_file)
+ });
+ }
+ let (mut distributions, mut estimates) = estimates(avg_times, config);
+ if sampling_mode.is_linear() {
+ let (distribution, slope) = regression(&data, config);
+
+ estimates.slope = Some(slope);
+ distributions.slope = Some(distribution);
+ }
+
+ if criterion.connection.is_none() && criterion.load_baseline.is_none() {
+ log_if_err!({
+ let mut sample_file = criterion.output_directory.clone();
+ sample_file.push(id.as_directory_name());
+ sample_file.push("new");
+ sample_file.push("sample.json");
+ fs::save(
+ &SavedSample {
+ sampling_mode,
+ iters: data.x().as_ref().to_vec(),
+ times: data.y().as_ref().to_vec(),
+ },
+ &sample_file,
+ )
+ });
+ log_if_err!({
+ let mut estimates_file = criterion.output_directory.clone();
+ estimates_file.push(id.as_directory_name());
+ estimates_file.push("new");
+ estimates_file.push("estimates.json");
+ fs::save(&estimates, &estimates_file)
+ });
+ }
+
+ let compare_data = if base_dir_exists(
+ id,
+ &criterion.baseline_directory,
+ &criterion.output_directory,
+ ) {
+ let result = compare::common(id, avg_times, config, criterion);
+ match result {
+ Ok((
+ t_value,
+ t_distribution,
+ relative_estimates,
+ relative_distributions,
+ base_iter_counts,
+ base_sample_times,
+ base_avg_times,
+ base_estimates,
+ )) => {
+ let p_value = t_distribution.p_value(t_value, &Tails::Two);
+ Some(crate::report::ComparisonData {
+ p_value,
+ t_distribution,
+ t_value,
+ relative_estimates,
+ relative_distributions,
+ significance_threshold: config.significance_level,
+ noise_threshold: config.noise_threshold,
+ base_iter_counts,
+ base_sample_times,
+ base_avg_times,
+ base_estimates,
+ })
+ }
+ Err(e) => {
+ crate::error::log_error(&e);
+ None
+ }
+ }
+ } else {
+ None
+ };
+
+ let measurement_data = crate::report::MeasurementData {
+ data: Data::new(&*iters, &*times),
+ avg_times: labeled_sample,
+ absolute_estimates: estimates,
+ distributions,
+ comparison: compare_data,
+ throughput,
+ };
+
+ criterion.report.measurement_complete(
+ id,
+ report_context,
+ &measurement_data,
+ criterion.measurement.formatter(),
+ );
+
+ if criterion.connection.is_none() && criterion.load_baseline.is_none() {
+ log_if_err!({
+ let mut benchmark_file = criterion.output_directory.clone();
+ benchmark_file.push(id.as_directory_name());
+ benchmark_file.push("new");
+ benchmark_file.push("benchmark.json");
+ fs::save(&id, &benchmark_file)
+ });
+ }
+
+ if criterion.connection.is_none() {
+ if let Baseline::Save = criterion.baseline {
+ copy_new_dir_to_base(
+ id.as_directory_name(),
+ &criterion.baseline_directory,
+ &criterion.output_directory,
+ );
+ }
+ }
+}
+
+fn base_dir_exists(id: &BenchmarkId, baseline: &str, output_directory: &Path) -> bool {
+ let mut base_dir = output_directory.to_owned();
+ base_dir.push(id.as_directory_name());
+ base_dir.push(baseline);
+ base_dir.exists()
+}
+
+// Performs a simple linear regression on the sample
+fn regression(
+ data: &Data<'_, f64, f64>,
+ config: &BenchmarkConfig,
+) -> (Distribution<f64>, Estimate) {
+ let cl = config.confidence_level;
+
+ let distribution = elapsed!(
+ "Bootstrapped linear regression",
+ data.bootstrap(config.nresamples, |d| (Slope::fit(&d).0,))
+ )
+ .0;
+
+ let point = Slope::fit(&data);
+ let (lb, ub) = distribution.confidence_interval(config.confidence_level);
+ let se = distribution.std_dev(None);
+
+ (
+ distribution,
+ Estimate {
+ confidence_interval: ConfidenceInterval {
+ confidence_level: cl,
+ lower_bound: lb,
+ upper_bound: ub,
+ },
+ point_estimate: point.0,
+ standard_error: se,
+ },
+ )
+}
+
+// Estimates the statistics of the population from the sample
+fn estimates(avg_times: &Sample<f64>, config: &BenchmarkConfig) -> (Distributions, Estimates) {
+ fn stats(sample: &Sample<f64>) -> (f64, f64, f64, f64) {
+ let mean = sample.mean();
+ let std_dev = sample.std_dev(Some(mean));
+ let median = sample.percentiles().median();
+ let mad = sample.median_abs_dev(Some(median));
+
+ (mean, std_dev, median, mad)
+ }
+
+ let cl = config.confidence_level;
+ let nresamples = config.nresamples;
+
+ let (mean, std_dev, median, mad) = stats(avg_times);
+ let points = PointEstimates {
+ mean,
+ median,
+ std_dev,
+ median_abs_dev: mad,
+ };
+
+ let (dist_mean, dist_stddev, dist_median, dist_mad) = elapsed!(
+ "Bootstrapping the absolute statistics.",
+ avg_times.bootstrap(nresamples, stats)
+ );
+
+ let distributions = Distributions {
+ mean: dist_mean,
+ slope: None,
+ median: dist_median,
+ median_abs_dev: dist_mad,
+ std_dev: dist_stddev,
+ };
+
+ let estimates = build_estimates(&distributions, &points, cl);
+
+ (distributions, estimates)
+}
+
+fn copy_new_dir_to_base(id: &str, baseline: &str, output_directory: &Path) {
+ let root_dir = Path::new(output_directory).join(id);
+ let base_dir = root_dir.join(baseline);
+ let new_dir = root_dir.join("new");
+
+ if !new_dir.exists() {
+ return;
+ };
+ if !base_dir.exists() {
+ try_else_return!(fs::mkdirp(&base_dir));
+ }
+
+ // TODO: consider using walkdir or similar to generically copy.
+ try_else_return!(fs::cp(
+ &new_dir.join("estimates.json"),
+ &base_dir.join("estimates.json")
+ ));
+ try_else_return!(fs::cp(
+ &new_dir.join("sample.json"),
+ &base_dir.join("sample.json")
+ ));
+ try_else_return!(fs::cp(
+ &new_dir.join("tukey.json"),
+ &base_dir.join("tukey.json")
+ ));
+ try_else_return!(fs::cp(
+ &new_dir.join("benchmark.json"),
+ &base_dir.join("benchmark.json")
+ ));
+ try_else_return!(fs::cp(&new_dir.join("raw.csv"), &base_dir.join("raw.csv")));
+}
diff --git a/src/benchmark.rs b/src/benchmark.rs
new file mode 100755
index 0000000..a99750d
--- /dev/null
+++ b/src/benchmark.rs
@@ -0,0 +1,611 @@
+use crate::analysis;
+use crate::connection::OutgoingMessage;
+use crate::measurement::{Measurement, WallTime};
+use crate::report::{BenchmarkId, ReportContext};
+use crate::routine::{Function, Routine};
+use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
+use std::cell::RefCell;
+use std::fmt::Debug;
+use std::marker::Sized;
+use std::time::Duration;
+
+// TODO: Move the benchmark config stuff to a separate module for easier use.
+
+/// Struct containing all of the configuration options for a benchmark.
+pub struct BenchmarkConfig {
+ pub confidence_level: f64,
+ pub measurement_time: Duration,
+ pub noise_threshold: f64,
+ pub nresamples: usize,
+ pub sample_size: usize,
+ pub significance_level: f64,
+ pub warm_up_time: Duration,
+ pub sampling_mode: SamplingMode,
+}
+
+/// Struct representing a partially-complete per-benchmark configuration.
+#[derive(Clone)]
+pub(crate) struct PartialBenchmarkConfig {
+ pub(crate) confidence_level: Option<f64>,
+ pub(crate) measurement_time: Option<Duration>,
+ pub(crate) noise_threshold: Option<f64>,
+ pub(crate) nresamples: Option<usize>,
+ pub(crate) sample_size: Option<usize>,
+ pub(crate) significance_level: Option<f64>,
+ pub(crate) warm_up_time: Option<Duration>,
+ pub(crate) sampling_mode: Option<SamplingMode>,
+ pub(crate) plot_config: PlotConfiguration,
+}
+
+impl Default for PartialBenchmarkConfig {
+ fn default() -> Self {
+ PartialBenchmarkConfig {
+ confidence_level: None,
+ measurement_time: None,
+ noise_threshold: None,
+ nresamples: None,
+ sample_size: None,
+ significance_level: None,
+ warm_up_time: None,
+ plot_config: PlotConfiguration::default(),
+ sampling_mode: None,
+ }
+ }
+}
+
+impl PartialBenchmarkConfig {
+ pub(crate) fn to_complete(&self, defaults: &BenchmarkConfig) -> BenchmarkConfig {
+ BenchmarkConfig {
+ confidence_level: self.confidence_level.unwrap_or(defaults.confidence_level),
+ measurement_time: self.measurement_time.unwrap_or(defaults.measurement_time),
+ noise_threshold: self.noise_threshold.unwrap_or(defaults.noise_threshold),
+ nresamples: self.nresamples.unwrap_or(defaults.nresamples),
+ sample_size: self.sample_size.unwrap_or(defaults.sample_size),
+ significance_level: self
+ .significance_level
+ .unwrap_or(defaults.significance_level),
+ warm_up_time: self.warm_up_time.unwrap_or(defaults.warm_up_time),
+ sampling_mode: self.sampling_mode.unwrap_or(defaults.sampling_mode),
+ }
+ }
+}
+
+pub(crate) struct NamedRoutine<T, M: Measurement = WallTime> {
+ pub id: String,
+ pub(crate) f: Box<RefCell<dyn Routine<M, T>>>,
+}
+
+/// Structure representing a benchmark (or group of benchmarks)
+/// which take one parameter.
+#[doc(hidden)]
+pub struct ParameterizedBenchmark<T: Debug, M: Measurement = WallTime> {
+ config: PartialBenchmarkConfig,
+ values: Vec<T>,
+ routines: Vec<NamedRoutine<T, M>>,
+ throughput: Option<Box<dyn Fn(&T) -> Throughput>>,
+}
+
+/// Structure representing a benchmark (or group of benchmarks)
+/// which takes no parameters.
+#[doc(hidden)]
+pub struct Benchmark<M: Measurement = WallTime> {
+ config: PartialBenchmarkConfig,
+ routines: Vec<NamedRoutine<(), M>>,
+ throughput: Option<Throughput>,
+}
+
+/// Common trait for `Benchmark` and `ParameterizedBenchmark`. Not intended to be
+/// used outside of Criterion.rs.
+#[doc(hidden)]
+pub trait BenchmarkDefinition<M: Measurement = WallTime>: Sized {
+ #[doc(hidden)]
+ fn run(self, group_id: &str, c: &mut Criterion<M>);
+}
+
+macro_rules! benchmark_config {
+ ($type:tt) => {
+ /// Changes the size of the sample for this benchmark
+ ///
+ /// A bigger sample should yield more accurate results if paired with a sufficiently large
+ /// measurement time.
+ ///
+ /// Sample size must be at least 10.
+ ///
+ /// # Panics
+ ///
+ /// Panics if n < 10.
+ pub fn sample_size(mut self, n: usize) -> Self {
+ assert!(n >= 10);
+
+ self.config.sample_size = Some(n);
+ self
+ }
+
+ /// Changes the warm up time for this benchmark
+ ///
+ /// # Panics
+ ///
+ /// Panics if the input duration is zero
+ pub fn warm_up_time(mut self, dur: Duration) -> Self {
+ assert!(dur.to_nanos() > 0);
+
+ self.config.warm_up_time = Some(dur);
+ self
+ }
+
+ /// Changes the target measurement time for this benchmark. Criterion will attempt
+ /// to spent approximately this amount of time measuring the benchmark.
+ /// With a longer time, the measurement will become more resilient to transitory peak loads
+ /// caused by external programs.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the input duration in zero
+ pub fn measurement_time(mut self, dur: Duration) -> Self {
+ assert!(dur.to_nanos() > 0);
+
+ self.config.measurement_time = Some(dur);
+ self
+ }
+
+ /// Changes the number of resamples for this benchmark
+ ///
+ /// Number of resamples to use for the
+ /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling)
+ ///
+ /// A larger number of resamples reduces the random sampling errors, which are inherent to the
+ /// bootstrap method, but also increases the analysis time.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the number of resamples is set to zero
+ pub fn nresamples(mut self, n: usize) -> Self {
+ assert!(n > 0);
+ if n <= 1000 {
+ println!("\nWarning: It is not recommended to reduce nresamples below 1000.");
+ }
+
+ self.config.nresamples = Some(n);
+ self
+ }
+
+ /// Changes the default noise threshold for this benchmark. The noise threshold
+ /// is used to filter out small changes in performance, even if they are statistically
+ /// significant. Sometimes benchmarking the same code twice will result in small but
+ /// statistically significant differences solely because of noise. This provides a way to filter
+ /// out some of these false positives at the cost of making it harder to detect small changes
+ /// to the true performance of the benchmark.
+ ///
+ /// The default is 0.01, meaning that changes smaller than 1% will be ignored.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the threshold is set to a negative value
+ pub fn noise_threshold(mut self, threshold: f64) -> Self {
+ assert!(threshold >= 0.0);
+
+ self.config.noise_threshold = Some(threshold);
+ self
+ }
+
+ /// Changes the default confidence level for this benchmark. The confidence
+ /// level is the desired probability that the true runtime lies within the estimated
+ /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is
+ /// 0.95, meaning that the confidence interval should capture the true value 95% of the time.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the confidence level is set to a value outside the `(0, 1)` range
+ pub fn confidence_level(mut self, cl: f64) -> Self {
+ assert!(cl > 0.0 && cl < 1.0);
+ if cl < 0.5 {
+ println!("\nWarning: It is not recommended to reduce confidence level below 0.5.");
+ }
+
+ self.config.confidence_level = Some(cl);
+ self
+ }
+
+ /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance)
+ /// for this benchmark. This is used to perform a
+ /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if
+ /// the measurements from this run are different from the measured performance of the last run.
+ /// The significance level is the desired probability that two measurements of identical code
+ /// will be considered 'different' due to noise in the measurements. The default value is 0.05,
+ /// meaning that approximately 5% of identical benchmarks will register as different due to
+ /// noise.
+ ///
+ /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
+ /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to
+ /// detect small but real changes in the performance. By setting the significance level
+ /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
+ /// report more spurious differences.
+ ///
+ /// See also the noise threshold setting.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the significance level is set to a value outside the `(0, 1)` range
+ pub fn significance_level(mut self, sl: f64) -> Self {
+ assert!(sl > 0.0 && sl < 1.0);
+
+ self.config.significance_level = Some(sl);
+ self
+ }
+
+ /// Changes the plot configuration for this benchmark.
+ pub fn plot_config(mut self, new_config: PlotConfiguration) -> Self {
+ self.config.plot_config = new_config;
+ self
+ }
+
+ /// Changes the sampling mode for this benchmark.
+ pub fn sampling_mode(mut self, new_mode: SamplingMode) -> Self {
+ self.config.sampling_mode = Some(new_mode);
+ self
+ }
+ };
+}
+
+impl<M> Benchmark<M>
+where
+ M: Measurement + 'static,
+{
+ benchmark_config!(Benchmark);
+
+ /// Create a new benchmark group and adds the given function to it.
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// # #[macro_use] extern crate criterion;
+ /// # use criterion::*;
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// // One-time setup goes here
+ /// c.bench(
+ /// "my_group",
+ /// Benchmark::new("my_function", |b| b.iter(|| {
+ /// // Code to benchmark goes here
+ /// })),
+ /// );
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ pub fn new<S, F>(id: S, f: F) -> Benchmark<M>
+ where
+ S: Into<String>,
+ F: FnMut(&mut Bencher<'_, M>) + 'static,
+ {
+ Benchmark {
+ config: PartialBenchmarkConfig::default(),
+ routines: vec![],
+ throughput: None,
+ }
+ .with_function(id, f)
+ }
+
+ /// Add a function to the benchmark group.
+ pub fn with_function<S, F>(mut self, id: S, mut f: F) -> Benchmark<M>
+ where
+ S: Into<String>,
+ F: FnMut(&mut Bencher<'_, M>) + 'static,
+ {
+ let routine = NamedRoutine {
+ id: id.into(),
+ f: Box::new(RefCell::new(Function::new(move |b, _| f(b)))),
+ };
+ self.routines.push(routine);
+ self
+ }
+
+ /// Set the input size for this benchmark group. Used for reporting the
+ /// throughput.
+ pub fn throughput(mut self, throughput: Throughput) -> Benchmark<M> {
+ self.throughput = Some(throughput);
+ self
+ }
+}
+
+impl<M: Measurement> BenchmarkDefinition<M> for Benchmark<M> {
+ fn run(self, group_id: &str, c: &mut Criterion<M>) {
+ let report_context = ReportContext {
+ output_directory: c.output_directory.clone(),
+ plot_config: self.config.plot_config.clone(),
+ };
+
+ let config = self.config.to_complete(&c.config);
+ let num_routines = self.routines.len();
+
+ let mut all_ids = vec![];
+ let mut any_matched = false;
+
+ if let Some(conn) = &c.connection {
+ conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id })
+ .unwrap();
+ }
+
+ for routine in self.routines {
+ let function_id = if num_routines == 1 && group_id == routine.id {
+ None
+ } else {
+ Some(routine.id)
+ };
+
+ let mut id = BenchmarkId::new(
+ group_id.to_owned(),
+ function_id,
+ None,
+ self.throughput.clone(),
+ );
+
+ id.ensure_directory_name_unique(&c.all_directories);
+ c.all_directories.insert(id.as_directory_name().to_owned());
+ id.ensure_title_unique(&c.all_titles);
+ c.all_titles.insert(id.as_title().to_owned());
+
+ let do_run = c.filter_matches(id.id());
+ any_matched |= do_run;
+
+ execute_benchmark(
+ do_run,
+ &id,
+ c,
+ &config,
+ &mut *routine.f.borrow_mut(),
+ &report_context,
+ &(),
+ self.throughput.clone(),
+ );
+
+ all_ids.push(id);
+ }
+
+ if let Some(conn) = &c.connection {
+ conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id })
+ .unwrap();
+ conn.serve_value_formatter(c.measurement.formatter())
+ .unwrap();
+ }
+
+ if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() {
+ c.report
+ .summarize(&report_context, &all_ids, c.measurement.formatter());
+ }
+ if any_matched {
+ c.report.group_separator();
+ }
+ }
+}
+
+impl<T, M> ParameterizedBenchmark<T, M>
+where
+ T: Debug + 'static,
+ M: Measurement + 'static,
+{
+ benchmark_config!(ParameterizedBenchmark);
+
+ pub(crate) fn with_functions(
+ functions: Vec<NamedRoutine<T, M>>,
+ parameters: Vec<T>,
+ ) -> ParameterizedBenchmark<T, M> {
+ ParameterizedBenchmark {
+ config: PartialBenchmarkConfig::default(),
+ values: parameters,
+ routines: functions,
+ throughput: None,
+ }
+ }
+
+ /// Create a new parameterized benchmark group and adds the given function
+ /// to it.
+ /// The function under test must follow the setup - bench - teardown pattern:
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// # #[macro_use] extern crate criterion;
+ /// # use criterion::*;
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let parameters = vec![1u64, 2u64, 3u64];
+ ///
+ /// // One-time setup goes here
+ /// c.bench(
+ /// "my_group",
+ /// ParameterizedBenchmark::new(
+ /// "my_function",
+ /// |b, param| b.iter(|| {
+ /// // Code to benchmark using param goes here
+ /// }),
+ /// parameters
+ /// )
+ /// );
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ pub fn new<S, F, I>(id: S, f: F, parameters: I) -> ParameterizedBenchmark<T, M>
+ where
+ S: Into<String>,
+ F: FnMut(&mut Bencher<'_, M>, &T) + 'static,
+ I: IntoIterator<Item = T>,
+ {
+ ParameterizedBenchmark {
+ config: PartialBenchmarkConfig::default(),
+ values: parameters.into_iter().collect(),
+ routines: vec![],
+ throughput: None,
+ }
+ .with_function(id, f)
+ }
+
+ /// Add a function to the benchmark group.
+ pub fn with_function<S, F>(mut self, id: S, f: F) -> ParameterizedBenchmark<T, M>
+ where
+ S: Into<String>,
+ F: FnMut(&mut Bencher<'_, M>, &T) + 'static,
+ {
+ let routine = NamedRoutine {
+ id: id.into(),
+ f: Box::new(RefCell::new(Function::new(f))),
+ };
+ self.routines.push(routine);
+ self
+ }
+
+ /// Use the given function to calculate the input size for a given input.
+ pub fn throughput<F>(mut self, throughput: F) -> ParameterizedBenchmark<T, M>
+ where
+ F: Fn(&T) -> Throughput + 'static,
+ {
+ self.throughput = Some(Box::new(throughput));
+ self
+ }
+}
+impl<T, M> BenchmarkDefinition<M> for ParameterizedBenchmark<T, M>
+where
+ T: Debug + 'static,
+ M: Measurement + 'static,
+{
+ fn run(self, group_id: &str, c: &mut Criterion<M>) {
+ let report_context = ReportContext {
+ output_directory: c.output_directory.clone(),
+ plot_config: self.config.plot_config.clone(),
+ };
+
+ let config = self.config.to_complete(&c.config);
+ let num_parameters = self.values.len();
+ let num_routines = self.routines.len();
+
+ let mut all_ids = vec![];
+ let mut any_matched = false;
+
+ if let Some(conn) = &c.connection {
+ conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id })
+ .unwrap();
+ }
+
+ for routine in self.routines {
+ for value in &self.values {
+ let function_id = if num_routines == 1 && group_id == routine.id {
+ None
+ } else {
+ Some(routine.id.clone())
+ };
+
+ let value_str = if num_parameters == 1 {
+ None
+ } else {
+ Some(format!("{:?}", value))
+ };
+
+ let throughput = self.throughput.as_ref().map(|func| func(value));
+ let mut id = BenchmarkId::new(
+ group_id.to_owned(),
+ function_id,
+ value_str,
+ throughput.clone(),
+ );
+
+ id.ensure_directory_name_unique(&c.all_directories);
+ c.all_directories.insert(id.as_directory_name().to_owned());
+ id.ensure_title_unique(&c.all_titles);
+ c.all_titles.insert(id.as_title().to_owned());
+
+ let do_run = c.filter_matches(id.id());
+ any_matched |= do_run;
+
+ execute_benchmark(
+ do_run,
+ &id,
+ c,
+ &config,
+ &mut *routine.f.borrow_mut(),
+ &report_context,
+ value,
+ throughput,
+ );
+
+ all_ids.push(id);
+ }
+ }
+
+ if let Some(conn) = &c.connection {
+ conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id })
+ .unwrap();
+ conn.serve_value_formatter(c.measurement.formatter())
+ .unwrap();
+ }
+
+ if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() {
+ c.report
+ .summarize(&report_context, &all_ids, c.measurement.formatter());
+ }
+ if any_matched {
+ c.report.group_separator();
+ }
+ }
+}
+
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::too_many_arguments))]
+fn execute_benchmark<T, M>(
+ do_run: bool,
+ id: &BenchmarkId,
+ c: &Criterion<M>,
+ config: &BenchmarkConfig,
+ routine: &mut dyn Routine<M, T>,
+ report_context: &ReportContext,
+ parameter: &T,
+ throughput: Option<Throughput>,
+) where
+ T: Debug,
+ M: Measurement,
+{
+ match c.mode {
+ Mode::Benchmark => {
+ if let Some(conn) = &c.connection {
+ if do_run {
+ conn.send(&OutgoingMessage::BeginningBenchmark { id: id.into() })
+ .unwrap();
+ } else {
+ conn.send(&OutgoingMessage::SkippingBenchmark { id: id.into() })
+ .unwrap();
+ }
+ }
+
+ if do_run {
+ analysis::common(
+ id,
+ routine,
+ config,
+ c,
+ report_context,
+ parameter,
+ throughput,
+ );
+ }
+ }
+ Mode::List => {
+ if do_run {
+ println!("{}: bench", id);
+ }
+ }
+ Mode::Test => {
+ if do_run {
+ // In test mode, run the benchmark exactly once, then exit.
+ c.report.test_start(id, report_context);
+ routine.test(&c.measurement, parameter);
+ c.report.test_pass(id, report_context);
+ }
+ }
+ Mode::Profile(duration) => {
+ if do_run {
+ routine.profile(&c.measurement, id, c, report_context, duration, parameter);
+ }
+ }
+ }
+}
diff --git a/src/benchmark_group.rs b/src/benchmark_group.rs
new file mode 100755
index 0000000..c59cf56
--- /dev/null
+++ b/src/benchmark_group.rs
@@ -0,0 +1,497 @@
+use crate::analysis;
+use crate::benchmark::PartialBenchmarkConfig;
+use crate::connection::OutgoingMessage;
+use crate::measurement::Measurement;
+use crate::report::BenchmarkId as InternalBenchmarkId;
+use crate::report::ReportContext;
+use crate::routine::{Function, Routine};
+use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
+use std::time::Duration;
+
+/// Structure used to group together a set of related benchmarks, along with custom configuration
+/// settings for groups of benchmarks. All benchmarks performed using a benchmark group will be
+/// grouped together in the final report.
+///
+/// # Examples:
+///
+/// ```no_run
+/// #[macro_use] extern crate criterion;
+/// use self::criterion::*;
+/// use std::time::Duration;
+///
+/// fn bench_simple(c: &mut Criterion) {
+/// let mut group = c.benchmark_group("My Group");
+///
+/// // Now we can perform benchmarks with this group
+/// group.bench_function("Bench 1", |b| b.iter(|| 1 ));
+/// group.bench_function("Bench 2", |b| b.iter(|| 2 ));
+///
+/// // It's recommended to call group.finish() explicitly at the end, but if you don't it will
+/// // be called automatically when the group is dropped.
+/// group.finish();
+/// }
+///
+/// fn bench_nested(c: &mut Criterion) {
+/// let mut group = c.benchmark_group("My Second Group");
+/// // We can override the configuration on a per-group level
+/// group.measurement_time(Duration::from_secs(1));
+///
+/// // We can also use loops to define multiple benchmarks, even over multiple dimensions.
+/// for x in 0..3 {
+/// for y in 0..3 {
+/// let point = (x, y);
+/// let parameter_string = format!("{} * {}", x, y);
+/// group.bench_with_input(BenchmarkId::new("Multiply", parameter_string), &point,
+/// |b, (p_x, p_y)| b.iter(|| p_x * p_y));
+/// }
+/// }
+///
+/// group.finish();
+/// }
+///
+/// fn bench_throughput(c: &mut Criterion) {
+/// let mut group = c.benchmark_group("Summation");
+///
+/// for size in [1024, 2048, 4096].iter() {
+/// // Generate input of an appropriate size...
+/// let input = vec![1u64, *size];
+///
+/// // We can use the throughput function to tell Criterion.rs how large the input is
+/// // so it can calculate the overall throughput of the function. If we wanted, we could
+/// // even change the benchmark configuration for different inputs (eg. to reduce the
+/// // number of samples for extremely large and slow inputs) or even different functions.
+/// group.throughput(Throughput::Elements(*size as u64));
+///
+/// group.bench_with_input(BenchmarkId::new("sum", *size), &input,
+/// |b, i| b.iter(|| i.iter().sum::<u64>()));
+/// group.bench_with_input(BenchmarkId::new("fold", *size), &input,
+/// |b, i| b.iter(|| i.iter().fold(0u64, |a, b| a + b)));
+/// }
+///
+/// group.finish();
+/// }
+///
+/// criterion_group!(benches, bench_simple, bench_nested, bench_throughput);
+/// criterion_main!(benches);
+/// ```
+pub struct BenchmarkGroup<'a, M: Measurement> {
+ criterion: &'a mut Criterion<M>,
+ group_name: String,
+ all_ids: Vec<InternalBenchmarkId>,
+ any_matched: bool,
+ partial_config: PartialBenchmarkConfig,
+ throughput: Option<Throughput>,
+}
+impl<'a, M: Measurement> BenchmarkGroup<'a, M> {
+ /// Changes the size of the sample for this benchmark
+ ///
+ /// A bigger sample should yield more accurate results if paired with a sufficiently large
+ /// measurement time.
+ ///
+ /// Sample size must be at least 10.
+ ///
+ /// # Panics
+ ///
+ /// Panics if n < 10.
+ pub fn sample_size(&mut self, n: usize) -> &mut Self {
+ assert!(n >= 10);
+
+ self.partial_config.sample_size = Some(n);
+ self
+ }
+
+ /// Changes the warm up time for this benchmark
+ ///
+ /// # Panics
+ ///
+ /// Panics if the input duration is zero
+ pub fn warm_up_time(&mut self, dur: Duration) -> &mut Self {
+ assert!(dur.to_nanos() > 0);
+
+ self.partial_config.warm_up_time = Some(dur);
+ self
+ }
+
+ /// Changes the target measurement time for this benchmark group.
+ ///
+ /// Criterion will attempt to spent approximately this amount of time measuring each
+ /// benchmark on a best-effort basis. If it is not possible to perform the measurement in
+ /// the requested time (eg. because each iteration of the benchmark is long) then Criterion
+ /// will spend as long as is needed to collect the desired number of samples. With a longer
+ /// time, the measurement will become more resilient to interference from other programs.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the input duration is zero
+ pub fn measurement_time(&mut self, dur: Duration) -> &mut Self {
+ assert!(dur.to_nanos() > 0);
+
+ self.partial_config.measurement_time = Some(dur);
+ self
+ }
+
+ /// Changes the number of resamples for this benchmark group
+ ///
+ /// Number of resamples to use for the
+ /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling)
+ ///
+ /// A larger number of resamples reduces the random sampling errors which are inherent to the
+ /// bootstrap method, but also increases the analysis time.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the number of resamples is set to zero
+ pub fn nresamples(&mut self, n: usize) -> &mut Self {
+ assert!(n > 0);
+ if n <= 1000 {
+ println!("\nWarning: It is not recommended to reduce nresamples below 1000.");
+ }
+
+ self.partial_config.nresamples = Some(n);
+ self
+ }
+
+ /// Changes the noise threshold for benchmarks in this group. The noise threshold
+ /// is used to filter out small changes in performance from one run to the next, even if they
+ /// are statistically significant. Sometimes benchmarking the same code twice will result in
+ /// small but statistically significant differences solely because of noise. This provides a way
+ /// to filter out some of these false positives at the cost of making it harder to detect small
+ /// changes to the true performance of the benchmark.
+ ///
+ /// The default is 0.01, meaning that changes smaller than 1% will be ignored.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the threshold is set to a negative value
+ pub fn noise_threshold(&mut self, threshold: f64) -> &mut Self {
+ assert!(threshold >= 0.0);
+
+ self.partial_config.noise_threshold = Some(threshold);
+ self
+ }
+
+ /// Changes the confidence level for benchmarks in this group. The confidence
+ /// level is the desired probability that the true runtime lies within the estimated
+ /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is
+ /// 0.95, meaning that the confidence interval should capture the true value 95% of the time.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the confidence level is set to a value outside the `(0, 1)` range
+ pub fn confidence_level(&mut self, cl: f64) -> &mut Self {
+ assert!(cl > 0.0 && cl < 1.0);
+ if cl < 0.5 {
+ println!("\nWarning: It is not recommended to reduce confidence level below 0.5.");
+ }
+
+ self.partial_config.confidence_level = Some(cl);
+ self
+ }
+
+ /// Changes the [significance level](https://en.wikipedia.org/wiki/Statistical_significance)
+ /// for benchmarks in this group. This is used to perform a
+ /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if
+ /// the measurements from this run are different from the measured performance of the last run.
+ /// The significance level is the desired probability that two measurements of identical code
+ /// will be considered 'different' due to noise in the measurements. The default value is 0.05,
+ /// meaning that approximately 5% of identical benchmarks will register as different due to
+ /// noise.
+ ///
+ /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
+ /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to
+ /// detect small but real changes in the performance. By setting the significance level
+ /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
+ /// report more spurious differences.
+ ///
+ /// See also the noise threshold setting.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the significance level is set to a value outside the `(0, 1)` range
+ pub fn significance_level(&mut self, sl: f64) -> &mut Self {
+ assert!(sl > 0.0 && sl < 1.0);
+
+ self.partial_config.significance_level = Some(sl);
+ self
+ }
+
+ /// Changes the plot configuration for this benchmark group.
+ pub fn plot_config(&mut self, new_config: PlotConfiguration) -> &mut Self {
+ self.partial_config.plot_config = new_config;
+ self
+ }
+
+ /// Set the input size for this benchmark group. Used for reporting the
+ /// throughput.
+ pub fn throughput(&mut self, throughput: Throughput) -> &mut Self {
+ self.throughput = Some(throughput);
+ self
+ }
+
+ /// Set the sampling mode for this benchmark group.
+ pub fn sampling_mode(&mut self, new_mode: SamplingMode) -> &mut Self {
+ self.partial_config.sampling_mode = Some(new_mode);
+ self
+ }
+
+ pub(crate) fn new(criterion: &mut Criterion<M>, group_name: String) -> BenchmarkGroup<'_, M> {
+ BenchmarkGroup {
+ criterion,
+ group_name,
+ all_ids: vec![],
+ any_matched: false,
+ partial_config: PartialBenchmarkConfig::default(),
+ throughput: None,
+ }
+ }
+
+ /// Benchmark the given parameterless function inside this benchmark group.
+ pub fn bench_function<ID: IntoBenchmarkId, F>(&mut self, id: ID, mut f: F) -> &mut Self
+ where
+ F: FnMut(&mut Bencher<'_, M>),
+ {
+ self.run_bench(id.into_benchmark_id(), &(), |b, _| f(b));
+ self
+ }
+
+ /// Benchmark the given parameterized function inside this benchmark group.
+ pub fn bench_with_input<ID: IntoBenchmarkId, F, I>(
+ &mut self,
+ id: ID,
+ input: &I,
+ f: F,
+ ) -> &mut Self
+ where
+ F: FnMut(&mut Bencher<'_, M>, &I),
+ I: ?Sized,
+ {
+ self.run_bench(id.into_benchmark_id(), input, f);
+ self
+ }
+
+ fn run_bench<F, I>(&mut self, id: BenchmarkId, input: &I, f: F)
+ where
+ F: FnMut(&mut Bencher<'_, M>, &I),
+ I: ?Sized,
+ {
+ let config = self.partial_config.to_complete(&self.criterion.config);
+ let report_context = ReportContext {
+ output_directory: self.criterion.output_directory.clone(),
+ plot_config: self.partial_config.plot_config.clone(),
+ };
+
+ let mut id = InternalBenchmarkId::new(
+ self.group_name.clone(),
+ id.function_name,
+ id.parameter,
+ self.throughput.clone(),
+ );
+
+ assert!(
+ !self.all_ids.contains(&id),
+ "Benchmark IDs must be unique within a group."
+ );
+
+ id.ensure_directory_name_unique(&self.criterion.all_directories);
+ self.criterion
+ .all_directories
+ .insert(id.as_directory_name().to_owned());
+ id.ensure_title_unique(&self.criterion.all_titles);
+ self.criterion.all_titles.insert(id.as_title().to_owned());
+
+ let do_run = self.criterion.filter_matches(id.id());
+ self.any_matched |= do_run;
+ let mut func = Function::new(f);
+
+ match self.criterion.mode {
+ Mode::Benchmark => {
+ if let Some(conn) = &self.criterion.connection {
+ if do_run {
+ conn.send(&OutgoingMessage::BeginningBenchmark { id: (&id).into() })
+ .unwrap();
+ } else {
+ conn.send(&OutgoingMessage::SkippingBenchmark { id: (&id).into() })
+ .unwrap();
+ }
+ }
+ if do_run {
+ analysis::common(
+ &id,
+ &mut func,
+ &config,
+ self.criterion,
+ &report_context,
+ input,
+ self.throughput.clone(),
+ );
+ }
+ }
+ Mode::List => {
+ if do_run {
+ println!("{}: bench", id);
+ }
+ }
+ Mode::Test => {
+ if do_run {
+ // In test mode, run the benchmark exactly once, then exit.
+ self.criterion.report.test_start(&id, &report_context);
+ func.test(&self.criterion.measurement, input);
+ self.criterion.report.test_pass(&id, &report_context);
+ }
+ }
+ Mode::Profile(duration) => {
+ if do_run {
+ func.profile(
+ &self.criterion.measurement,
+ &id,
+ &self.criterion,
+ &report_context,
+ duration,
+ input,
+ );
+ }
+ }
+ }
+
+ self.all_ids.push(id);
+ }
+
+ /// Consume the benchmark group and generate the summary reports for the group.
+ ///
+ /// It is recommended to call this explicitly, but if you forget it will be called when the
+ /// group is dropped.
+ pub fn finish(self) {
+ ::std::mem::drop(self);
+ }
+}
+impl<'a, M: Measurement> Drop for BenchmarkGroup<'a, M> {
+ fn drop(&mut self) {
+ // I don't really like having a bunch of non-trivial code in drop, but this is the only way
+ // to really write linear types like this in Rust...
+ if let Some(conn) = &mut self.criterion.connection {
+ conn.send(&OutgoingMessage::FinishedBenchmarkGroup {
+ group: &self.group_name,
+ })
+ .unwrap();
+
+ conn.serve_value_formatter(self.criterion.measurement.formatter())
+ .unwrap();
+ }
+
+ if self.all_ids.len() > 1 && self.any_matched && self.criterion.mode.is_benchmark() {
+ let report_context = ReportContext {
+ output_directory: self.criterion.output_directory.clone(),
+ plot_config: self.partial_config.plot_config.clone(),
+ };
+
+ self.criterion.report.summarize(
+ &report_context,
+ &self.all_ids,
+ self.criterion.measurement.formatter(),
+ );
+ }
+ if self.any_matched {
+ self.criterion.report.group_separator();
+ }
+ }
+}
+
+/// Simple structure representing an ID for a benchmark. The ID must be unique within a benchmark
+/// group.
+#[derive(Clone, Eq, PartialEq, Hash)]
+pub struct BenchmarkId {
+ pub(crate) function_name: Option<String>,
+ pub(crate) parameter: Option<String>,
+}
+impl BenchmarkId {
+ /// Construct a new benchmark ID from a string function name and a parameter value.
+ ///
+ /// Note that the parameter value need not be the same as the parameter passed to your
+ /// actual benchmark. For instance, you might have a benchmark that takes a 1MB string as
+ /// input. It would be impractical to embed the whole string in the benchmark ID, so instead
+ /// your parameter value might be a descriptive string like "1MB Alphanumeric".
+ ///
+ /// # Examples
+ /// ```
+ /// # use criterion::{BenchmarkId, Criterion};
+ /// // A basic benchmark ID is typically constructed from a constant string and a simple
+ /// // parameter
+ /// let basic_id = BenchmarkId::new("my_id", 5);
+ ///
+ /// // The function name can be a string
+ /// let function_name = "test_string".to_string();
+ /// let string_id = BenchmarkId::new(function_name, 12);
+ ///
+ /// // Benchmark IDs are passed to benchmark groups:
+ /// let mut criterion = Criterion::default();
+ /// let mut group = criterion.benchmark_group("My Group");
+ /// // Generate a very large input
+ /// let input : String = ::std::iter::repeat("X").take(1024 * 1024).collect();
+ ///
+ /// // Note that we don't have to use the input as the parameter in the ID
+ /// group.bench_with_input(BenchmarkId::new("Test long string", "1MB X's"), &input, |b, i| {
+ /// b.iter(|| i.len())
+ /// });
+ /// ```
+ pub fn new<S: Into<String>, P: ::std::fmt::Display>(
+ function_name: S,
+ parameter: P,
+ ) -> BenchmarkId {
+ BenchmarkId {
+ function_name: Some(function_name.into()),
+ parameter: Some(format!("{}", parameter)),
+ }
+ }
+
+ /// Construct a new benchmark ID from just a parameter value. Use this when benchmarking a
+ /// single function with a variety of different inputs.
+ pub fn from_parameter<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId {
+ BenchmarkId {
+ function_name: None,
+ parameter: Some(format!("{}", parameter)),
+ }
+ }
+
+ pub(crate) fn no_function() -> BenchmarkId {
+ BenchmarkId {
+ function_name: None,
+ parameter: None,
+ }
+ }
+
+ pub(crate) fn no_function_with_input<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId {
+ BenchmarkId {
+ function_name: None,
+ parameter: Some(format!("{}", parameter)),
+ }
+ }
+}
+
+mod private {
+ pub trait Sealed {}
+ impl Sealed for super::BenchmarkId {}
+ impl<S: Into<String>> Sealed for S {}
+}
+
+/// Sealed trait which allows users to automatically convert strings to benchmark IDs.
+pub trait IntoBenchmarkId: private::Sealed {
+ fn into_benchmark_id(self) -> BenchmarkId;
+}
+impl IntoBenchmarkId for BenchmarkId {
+ fn into_benchmark_id(self) -> BenchmarkId {
+ self
+ }
+}
+impl<S: Into<String>> IntoBenchmarkId for S {
+ fn into_benchmark_id(self) -> BenchmarkId {
+ let function_name = self.into();
+ if function_name.is_empty() {
+ panic!("Function name must not be empty.");
+ }
+
+ BenchmarkId {
+ function_name: Some(function_name),
+ parameter: None,
+ }
+ }
+}
diff --git a/src/connection.rs b/src/connection.rs
new file mode 100755
index 0000000..1f42fff
--- /dev/null
+++ b/src/connection.rs
@@ -0,0 +1,370 @@
+use crate::report::BenchmarkId as InternalBenchmarkId;
+use crate::Throughput;
+use std::cell::RefCell;
+use std::convert::TryFrom;
+use std::io::{Read, Write};
+use std::mem::size_of;
+use std::net::TcpStream;
+
+#[derive(Debug)]
+pub enum MessageError {
+ SerializationError(serde_cbor::Error),
+ IoError(std::io::Error),
+}
+impl From<serde_cbor::Error> for MessageError {
+ fn from(other: serde_cbor::Error) -> Self {
+ MessageError::SerializationError(other)
+ }
+}
+impl From<std::io::Error> for MessageError {
+ fn from(other: std::io::Error) -> Self {
+ MessageError::IoError(other)
+ }
+}
+impl std::fmt::Display for MessageError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ MessageError::SerializationError(error) => write!(
+ f,
+ "Failed to serialize or deserialize message to Criterion.rs benchmark:\n{}",
+ error
+ ),
+ MessageError::IoError(error) => write!(
+ f,
+ "Failed to read or write message to Criterion.rs benchmark:\n{}",
+ error
+ ),
+ }
+ }
+}
+impl std::error::Error for MessageError {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ match self {
+ MessageError::SerializationError(err) => Some(err),
+ MessageError::IoError(err) => Some(err),
+ }
+ }
+}
+
+// Use str::len as a const fn once we bump MSRV over 1.39.
+const RUNNER_MAGIC_NUMBER: &str = "cargo-criterion";
+const RUNNER_HELLO_SIZE: usize = 15 //RUNNER_MAGIC_NUMBER.len() // magic number
+ + (size_of::<u8>() * 3); // version number
+
+const BENCHMARK_MAGIC_NUMBER: &str = "Criterion";
+const BENCHMARK_HELLO_SIZE: usize = 9 //BENCHMARK_MAGIC_NUMBER.len() // magic number
+ + (size_of::<u8>() * 3) // version number
+ + size_of::<u16>() // protocol version
+ + size_of::<u16>(); // protocol format
+const PROTOCOL_VERSION: u16 = 1;
+const PROTOCOL_FORMAT: u16 = 1;
+
+#[derive(Debug)]
+struct InnerConnection {
+ socket: TcpStream,
+ receive_buffer: Vec<u8>,
+ send_buffer: Vec<u8>,
+ runner_version: [u8; 3],
+}
+impl InnerConnection {
+ pub fn new(mut socket: TcpStream) -> Result<Self, std::io::Error> {
+ // read the runner-hello
+ let mut hello_buf = [0u8; RUNNER_HELLO_SIZE];
+ socket.read_exact(&mut hello_buf)?;
+ if &hello_buf[0..RUNNER_MAGIC_NUMBER.len()] != RUNNER_MAGIC_NUMBER.as_bytes() {
+ panic!("Not connected to cargo-criterion.");
+ }
+ let i = RUNNER_MAGIC_NUMBER.len();
+ let runner_version = [hello_buf[i], hello_buf[i + 1], hello_buf[i + 2]];
+
+ info!("Runner version: {:?}", runner_version);
+
+ // now send the benchmark-hello
+ let mut hello_buf = [0u8; BENCHMARK_HELLO_SIZE];
+ hello_buf[0..BENCHMARK_MAGIC_NUMBER.len()]
+ .copy_from_slice(BENCHMARK_MAGIC_NUMBER.as_bytes());
+ let mut i = BENCHMARK_MAGIC_NUMBER.len();
+ hello_buf[i] = env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap();
+ hello_buf[i + 1] = env!("CARGO_PKG_VERSION_MINOR").parse().unwrap();
+ hello_buf[i + 2] = env!("CARGO_PKG_VERSION_PATCH").parse().unwrap();
+ i += 3;
+ hello_buf[i..i + 2].clone_from_slice(&PROTOCOL_VERSION.to_be_bytes());
+ i += 2;
+ hello_buf[i..i + 2].clone_from_slice(&PROTOCOL_FORMAT.to_be_bytes());
+
+ socket.write_all(&hello_buf)?;
+
+ Ok(InnerConnection {
+ socket,
+ receive_buffer: vec![],
+ send_buffer: vec![],
+ runner_version,
+ })
+ }
+
+ #[allow(dead_code)]
+ pub fn recv(&mut self) -> Result<IncomingMessage, MessageError> {
+ let mut length_buf = [0u8; 4];
+ self.socket.read_exact(&mut length_buf)?;
+ let length = u32::from_be_bytes(length_buf);
+ self.receive_buffer.resize(length as usize, 0u8);
+ self.socket.read_exact(&mut self.receive_buffer)?;
+ let value = serde_cbor::from_slice(&self.receive_buffer)?;
+ Ok(value)
+ }
+
+ pub fn send(&mut self, message: &OutgoingMessage) -> Result<(), MessageError> {
+ self.send_buffer.truncate(0);
+ serde_cbor::to_writer(&mut self.send_buffer, message)?;
+ let size = u32::try_from(self.send_buffer.len()).unwrap();
+ let length_buf = size.to_be_bytes();
+ self.socket.write_all(&length_buf)?;
+ self.socket.write_all(&self.send_buffer)?;
+ Ok(())
+ }
+}
+
+/// This is really just a holder to allow us to send messages through a shared reference to the
+/// connection.
+#[derive(Debug)]
+pub struct Connection {
+ inner: RefCell<InnerConnection>,
+}
+impl Connection {
+ pub fn new(socket: TcpStream) -> Result<Self, std::io::Error> {
+ Ok(Connection {
+ inner: RefCell::new(InnerConnection::new(socket)?),
+ })
+ }
+
+ #[allow(dead_code)]
+ pub fn recv(&self) -> Result<IncomingMessage, MessageError> {
+ self.inner.borrow_mut().recv()
+ }
+
+ pub fn send(&self, message: &OutgoingMessage) -> Result<(), MessageError> {
+ self.inner.borrow_mut().send(message)
+ }
+
+ pub fn serve_value_formatter(
+ &self,
+ formatter: &dyn crate::measurement::ValueFormatter,
+ ) -> Result<(), MessageError> {
+ loop {
+ let response = match self.recv()? {
+ IncomingMessage::FormatValue { value } => OutgoingMessage::FormattedValue {
+ value: formatter.format_value(value),
+ },
+ IncomingMessage::FormatThroughput { value, throughput } => {
+ OutgoingMessage::FormattedValue {
+ value: formatter.format_throughput(&throughput, value),
+ }
+ }
+ IncomingMessage::ScaleValues {
+ typical_value,
+ mut values,
+ } => {
+ let unit = formatter.scale_values(typical_value, &mut values);
+ OutgoingMessage::ScaledValues {
+ unit,
+ scaled_values: values,
+ }
+ }
+ IncomingMessage::ScaleThroughputs {
+ typical_value,
+ throughput,
+ mut values,
+ } => {
+ let unit = formatter.scale_throughputs(typical_value, &throughput, &mut values);
+ OutgoingMessage::ScaledValues {
+ unit,
+ scaled_values: values,
+ }
+ }
+ IncomingMessage::ScaleForMachines { mut values } => {
+ let unit = formatter.scale_for_machines(&mut values);
+ OutgoingMessage::ScaledValues {
+ unit,
+ scaled_values: values,
+ }
+ }
+ IncomingMessage::Continue => break,
+ _ => panic!(),
+ };
+ self.send(&response)?;
+ }
+ Ok(())
+ }
+}
+
+/// Enum defining the messages we can receive
+#[derive(Debug, Deserialize)]
+pub enum IncomingMessage {
+ // Value formatter requests
+ FormatValue {
+ value: f64,
+ },
+ FormatThroughput {
+ value: f64,
+ throughput: Throughput,
+ },
+ ScaleValues {
+ typical_value: f64,
+ values: Vec<f64>,
+ },
+ ScaleThroughputs {
+ typical_value: f64,
+ values: Vec<f64>,
+ throughput: Throughput,
+ },
+ ScaleForMachines {
+ values: Vec<f64>,
+ },
+ Continue,
+
+ __Other,
+}
+
+/// Enum defining the messages we can send
+#[derive(Debug, Serialize)]
+pub enum OutgoingMessage<'a> {
+ BeginningBenchmarkGroup {
+ group: &'a str,
+ },
+ FinishedBenchmarkGroup {
+ group: &'a str,
+ },
+ BeginningBenchmark {
+ id: RawBenchmarkId,
+ },
+ SkippingBenchmark {
+ id: RawBenchmarkId,
+ },
+ Warmup {
+ id: RawBenchmarkId,
+ nanos: f64,
+ },
+ MeasurementStart {
+ id: RawBenchmarkId,
+ sample_count: u64,
+ estimate_ns: f64,
+ iter_count: u64,
+ },
+ MeasurementComplete {
+ id: RawBenchmarkId,
+ iters: &'a [f64],
+ times: &'a [f64],
+ plot_config: PlotConfiguration,
+ sampling_method: SamplingMethod,
+ benchmark_config: BenchmarkConfig,
+ },
+ // value formatter responses
+ FormattedValue {
+ value: String,
+ },
+ ScaledValues {
+ scaled_values: Vec<f64>,
+ unit: &'a str,
+ },
+}
+
+// Also define serializable variants of certain things, either to avoid leaking
+// serializability into the public interface or because the serialized form
+// is a bit different from the regular one.
+
+#[derive(Debug, Serialize)]
+pub struct RawBenchmarkId {
+ group_id: String,
+ function_id: Option<String>,
+ value_str: Option<String>,
+ throughput: Vec<Throughput>,
+}
+impl From<&InternalBenchmarkId> for RawBenchmarkId {
+ fn from(other: &InternalBenchmarkId) -> RawBenchmarkId {
+ RawBenchmarkId {
+ group_id: other.group_id.clone(),
+ function_id: other.function_id.clone(),
+ value_str: other.value_str.clone(),
+ throughput: other.throughput.iter().cloned().collect(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub enum AxisScale {
+ Linear,
+ Logarithmic,
+}
+impl From<crate::AxisScale> for AxisScale {
+ fn from(other: crate::AxisScale) -> Self {
+ match other {
+ crate::AxisScale::Linear => AxisScale::Linear,
+ crate::AxisScale::Logarithmic => AxisScale::Logarithmic,
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub struct PlotConfiguration {
+ summary_scale: AxisScale,
+}
+impl From<&crate::PlotConfiguration> for PlotConfiguration {
+ fn from(other: &crate::PlotConfiguration) -> Self {
+ PlotConfiguration {
+ summary_scale: other.summary_scale.into(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+struct Duration {
+ secs: u64,
+ nanos: u32,
+}
+impl From<std::time::Duration> for Duration {
+ fn from(other: std::time::Duration) -> Self {
+ Duration {
+ secs: other.as_secs(),
+ nanos: other.subsec_nanos(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub struct BenchmarkConfig {
+ confidence_level: f64,
+ measurement_time: Duration,
+ noise_threshold: f64,
+ nresamples: usize,
+ sample_size: usize,
+ significance_level: f64,
+ warm_up_time: Duration,
+}
+impl From<&crate::benchmark::BenchmarkConfig> for BenchmarkConfig {
+ fn from(other: &crate::benchmark::BenchmarkConfig) -> Self {
+ BenchmarkConfig {
+ confidence_level: other.confidence_level,
+ measurement_time: other.measurement_time.into(),
+ noise_threshold: other.noise_threshold,
+ nresamples: other.nresamples,
+ sample_size: other.sample_size,
+ significance_level: other.significance_level,
+ warm_up_time: other.warm_up_time.into(),
+ }
+ }
+}
+
+/// Currently not used; defined for forwards compatibility with cargo-criterion.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum SamplingMethod {
+ Linear,
+ Flat,
+}
+impl From<crate::ActualSamplingMode> for SamplingMethod {
+ fn from(other: crate::ActualSamplingMode) -> Self {
+ match other {
+ crate::ActualSamplingMode::Flat => SamplingMethod::Flat,
+ crate::ActualSamplingMode::Linear => SamplingMethod::Linear,
+ }
+ }
+}
diff --git a/src/csv_report.rs b/src/csv_report.rs
new file mode 100755
index 0000000..f30b817
--- /dev/null
+++ b/src/csv_report.rs
@@ -0,0 +1,90 @@
+use crate::error::Result;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, MeasurementData, Report, ReportContext};
+use crate::Throughput;
+use csv::Writer;
+use std::io::Write;
+use std::path::Path;
+
+#[derive(Serialize)]
+struct CsvRow<'a> {
+ group: &'a str,
+ function: Option<&'a str>,
+ value: Option<&'a str>,
+ throughput_num: Option<&'a str>,
+ throughput_type: Option<&'a str>,
+ sample_measured_value: f64,
+ unit: &'static str,
+ iteration_count: u64,
+}
+
+struct CsvReportWriter<W: Write> {
+ writer: Writer<W>,
+}
+impl<W: Write> CsvReportWriter<W> {
+ fn write_data(
+ &mut self,
+ id: &BenchmarkId,
+ data: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) -> Result<()> {
+ let mut data_scaled: Vec<f64> = data.sample_times().as_ref().into();
+ let unit = formatter.scale_for_machines(&mut data_scaled);
+ let group = id.group_id.as_str();
+ let function = id.function_id.as_ref().map(String::as_str);
+ let value = id.value_str.as_ref().map(String::as_str);
+ let (throughput_num, throughput_type) = match id.throughput {
+ Some(Throughput::Bytes(bytes)) => (Some(format!("{}", bytes)), Some("bytes")),
+ Some(Throughput::Elements(elems)) => (Some(format!("{}", elems)), Some("elements")),
+ None => (None, None),
+ };
+ let throughput_num = throughput_num.as_ref().map(String::as_str);
+
+ for (count, measured_value) in data.iter_counts().iter().zip(data_scaled.into_iter()) {
+ let row = CsvRow {
+ group,
+ function,
+ value,
+ throughput_num,
+ throughput_type,
+ sample_measured_value: measured_value,
+ unit,
+ iteration_count: (*count) as u64,
+ };
+ self.writer.serialize(row)?;
+ }
+ Ok(())
+ }
+}
+
+pub struct FileCsvReport;
+impl FileCsvReport {
+ fn write_file(
+ &self,
+ path: &Path,
+ id: &BenchmarkId,
+ measurements: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) -> Result<()> {
+ let writer = Writer::from_path(path)?;
+ let mut writer = CsvReportWriter { writer };
+ writer.write_data(id, measurements, formatter)?;
+ Ok(())
+ }
+}
+
+impl Report for FileCsvReport {
+ fn measurement_complete(
+ &self,
+ id: &BenchmarkId,
+ context: &ReportContext,
+ measurements: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) {
+ let mut path = context.output_directory.clone();
+ path.push(id.as_directory_name());
+ path.push("new");
+ path.push("raw.csv");
+ log_if_err!(self.write_file(&path, id, measurements, formatter));
+ }
+}
diff --git a/src/error.rs b/src/error.rs
new file mode 100755
index 0000000..c1ecf76
--- /dev/null
+++ b/src/error.rs
@@ -0,0 +1,72 @@
+use csv::Error as CsvError;
+use serde_json::Error as SerdeError;
+use std::error::Error as StdError;
+use std::fmt;
+use std::io;
+use std::path::PathBuf;
+
+#[derive(Debug)]
+pub enum Error {
+ AccessError {
+ path: PathBuf,
+ inner: io::Error,
+ },
+ CopyError {
+ from: PathBuf,
+ to: PathBuf,
+ inner: io::Error,
+ },
+ SerdeError {
+ path: PathBuf,
+ inner: SerdeError,
+ },
+ CsvError(CsvError),
+}
+impl fmt::Display for Error {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Error::AccessError { path, inner } => {
+ write!(f, "Failed to access file {:?}: {}", path, inner)
+ }
+ Error::CopyError { from, to, inner } => {
+ write!(f, "Failed to copy file {:?} to {:?}: {}", from, to, inner)
+ }
+ Error::SerdeError { path, inner } => write!(
+ f,
+ "Failed to read or write file {:?} due to serialization error: {}",
+ path, inner
+ ),
+ Error::CsvError(inner) => write!(f, "CSV error: {}", inner),
+ }
+ }
+}
+impl StdError for Error {
+ fn description(&self) -> &str {
+ match self {
+ Error::AccessError { .. } => "AccessError",
+ Error::CopyError { .. } => "CopyError",
+ Error::SerdeError { .. } => "SerdeError",
+ Error::CsvError(_) => "CsvError",
+ }
+ }
+
+ fn cause(&self) -> Option<&dyn StdError> {
+ match self {
+ Error::AccessError { inner, .. } => Some(inner),
+ Error::CopyError { inner, .. } => Some(inner),
+ Error::SerdeError { inner, .. } => Some(inner),
+ Error::CsvError(inner) => Some(inner),
+ }
+ }
+}
+impl From<CsvError> for Error {
+ fn from(other: CsvError) -> Error {
+ Error::CsvError(other)
+ }
+}
+
+pub type Result<T> = ::std::result::Result<T, Error>;
+
+pub(crate) fn log_error(e: &Error) {
+ error!("error: {}", e);
+}
diff --git a/src/estimate.rs b/src/estimate.rs
new file mode 100755
index 0000000..8a79d27
--- /dev/null
+++ b/src/estimate.rs
@@ -0,0 +1,184 @@
+use std::fmt;
+
+use crate::stats::Distribution;
+
+#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize, Debug)]
+pub enum Statistic {
+ Mean,
+ Median,
+ MedianAbsDev,
+ Slope,
+ StdDev,
+ Typical,
+}
+
+impl fmt::Display for Statistic {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match *self {
+ Statistic::Mean => f.pad("mean"),
+ Statistic::Median => f.pad("median"),
+ Statistic::MedianAbsDev => f.pad("MAD"),
+ Statistic::Slope => f.pad("slope"),
+ Statistic::StdDev => f.pad("SD"),
+ Statistic::Typical => f.pad("typical"),
+ }
+ }
+}
+
+#[derive(Clone, PartialEq, Deserialize, Serialize, Debug)]
+pub struct ConfidenceInterval {
+ pub confidence_level: f64,
+ pub lower_bound: f64,
+ pub upper_bound: f64,
+}
+
+#[derive(Clone, PartialEq, Deserialize, Serialize, Debug)]
+pub struct Estimate {
+ /// The confidence interval for this estimate
+ pub confidence_interval: ConfidenceInterval,
+ ///
+ pub point_estimate: f64,
+ /// The standard error of this estimate
+ pub standard_error: f64,
+}
+
+pub fn build_estimates(
+ distributions: &Distributions,
+ points: &PointEstimates,
+ cl: f64,
+) -> Estimates {
+ let to_estimate = |point_estimate, distribution: &Distribution<f64>| {
+ let (lb, ub) = distribution.confidence_interval(cl);
+
+ Estimate {
+ confidence_interval: ConfidenceInterval {
+ confidence_level: cl,
+ lower_bound: lb,
+ upper_bound: ub,
+ },
+ point_estimate,
+ standard_error: distribution.std_dev(None),
+ }
+ };
+
+ Estimates {
+ mean: to_estimate(points.mean, &distributions.mean),
+ median: to_estimate(points.median, &distributions.median),
+ median_abs_dev: to_estimate(points.median_abs_dev, &distributions.median_abs_dev),
+ slope: None,
+ std_dev: to_estimate(points.std_dev, &distributions.std_dev),
+ }
+}
+
+pub fn build_change_estimates(
+ distributions: &ChangeDistributions,
+ points: &ChangePointEstimates,
+ cl: f64,
+) -> ChangeEstimates {
+ let to_estimate = |point_estimate, distribution: &Distribution<f64>| {
+ let (lb, ub) = distribution.confidence_interval(cl);
+
+ Estimate {
+ confidence_interval: ConfidenceInterval {
+ confidence_level: cl,
+ lower_bound: lb,
+ upper_bound: ub,
+ },
+ point_estimate,
+ standard_error: distribution.std_dev(None),
+ }
+ };
+
+ ChangeEstimates {
+ mean: to_estimate(points.mean, &distributions.mean),
+ median: to_estimate(points.median, &distributions.median),
+ }
+}
+
+pub struct PointEstimates {
+ pub mean: f64,
+ pub median: f64,
+ pub median_abs_dev: f64,
+ pub std_dev: f64,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Estimates {
+ pub mean: Estimate,
+ pub median: Estimate,
+ pub median_abs_dev: Estimate,
+ pub slope: Option<Estimate>,
+ pub std_dev: Estimate,
+}
+impl Estimates {
+ pub fn typical(&self) -> &Estimate {
+ self.slope.as_ref().unwrap_or(&self.mean)
+ }
+ pub fn get(&self, stat: Statistic) -> Option<&Estimate> {
+ match stat {
+ Statistic::Mean => Some(&self.mean),
+ Statistic::Median => Some(&self.median),
+ Statistic::MedianAbsDev => Some(&self.median_abs_dev),
+ Statistic::Slope => self.slope.as_ref(),
+ Statistic::StdDev => Some(&self.std_dev),
+ Statistic::Typical => Some(self.typical()),
+ }
+ }
+}
+
+pub struct Distributions {
+ pub mean: Distribution<f64>,
+ pub median: Distribution<f64>,
+ pub median_abs_dev: Distribution<f64>,
+ pub slope: Option<Distribution<f64>>,
+ pub std_dev: Distribution<f64>,
+}
+impl Distributions {
+ pub fn typical(&self) -> &Distribution<f64> {
+ self.slope.as_ref().unwrap_or(&self.mean)
+ }
+ pub fn get(&self, stat: Statistic) -> Option<&Distribution<f64>> {
+ match stat {
+ Statistic::Mean => Some(&self.mean),
+ Statistic::Median => Some(&self.median),
+ Statistic::MedianAbsDev => Some(&self.median_abs_dev),
+ Statistic::Slope => self.slope.as_ref(),
+ Statistic::StdDev => Some(&self.std_dev),
+ Statistic::Typical => Some(self.typical()),
+ }
+ }
+}
+
+pub struct ChangePointEstimates {
+ pub mean: f64,
+ pub median: f64,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct ChangeEstimates {
+ pub mean: Estimate,
+ pub median: Estimate,
+}
+impl ChangeEstimates {
+ pub fn get(&self, stat: Statistic) -> &Estimate {
+ match stat {
+ Statistic::Mean => &self.mean,
+ Statistic::Median => &self.median,
+ _ => panic!("Unexpected statistic"),
+ }
+ }
+}
+
+pub struct ChangeDistributions {
+ pub mean: Distribution<f64>,
+ pub median: Distribution<f64>,
+}
+impl ChangeDistributions {
+ pub fn get(&self, stat: Statistic) -> &Distribution<f64> {
+ match stat {
+ Statistic::Mean => &self.mean,
+ Statistic::Median => &self.median,
+ _ => panic!("Unexpected statistic"),
+ }
+ }
+}
diff --git a/src/format.rs b/src/format.rs
new file mode 100755
index 0000000..984f0f5
--- /dev/null
+++ b/src/format.rs
@@ -0,0 +1,104 @@
+pub fn change(pct: f64, signed: bool) -> String {
+ if signed {
+ format!("{:>+6}%", signed_short(pct * 1e2))
+ } else {
+ format!("{:>6}%", short(pct * 1e2))
+ }
+}
+
+pub fn time(ns: f64) -> String {
+ if ns < 1.0 {
+ format!("{:>6} ps", short(ns * 1e3))
+ } else if ns < 10f64.powi(3) {
+ format!("{:>6} ns", short(ns))
+ } else if ns < 10f64.powi(6) {
+ format!("{:>6} us", short(ns / 1e3))
+ } else if ns < 10f64.powi(9) {
+ format!("{:>6} ms", short(ns / 1e6))
+ } else {
+ format!("{:>6} s", short(ns / 1e9))
+ }
+}
+
+pub fn short(n: f64) -> String {
+ if n < 10.0 {
+ format!("{:.4}", n)
+ } else if n < 100.0 {
+ format!("{:.3}", n)
+ } else if n < 1000.0 {
+ format!("{:.2}", n)
+ } else if n < 10000.0 {
+ format!("{:.1}", n)
+ } else {
+ format!("{:.0}", n)
+ }
+}
+
+fn signed_short(n: f64) -> String {
+ let n_abs = n.abs();
+
+ if n_abs < 10.0 {
+ format!("{:+.4}", n)
+ } else if n_abs < 100.0 {
+ format!("{:+.3}", n)
+ } else if n_abs < 1000.0 {
+ format!("{:+.2}", n)
+ } else if n_abs < 10000.0 {
+ format!("{:+.1}", n)
+ } else {
+ format!("{:+.0}", n)
+ }
+}
+
+pub fn iter_count(iterations: u64) -> String {
+ if iterations < 10_000 {
+ format!("{} iterations", iterations)
+ } else if iterations < 1_000_000 {
+ format!("{:.0}k iterations", (iterations as f64) / 1000.0)
+ } else if iterations < 10_000_000 {
+ format!("{:.1}M iterations", (iterations as f64) / (1000.0 * 1000.0))
+ } else if iterations < 1_000_000_000 {
+ format!("{:.0}M iterations", (iterations as f64) / (1000.0 * 1000.0))
+ } else if iterations < 10_000_000_000 {
+ format!(
+ "{:.1}B iterations",
+ (iterations as f64) / (1000.0 * 1000.0 * 1000.0)
+ )
+ } else {
+ format!(
+ "{:.0}B iterations",
+ (iterations as f64) / (1000.0 * 1000.0 * 1000.0)
+ )
+ }
+}
+
+pub fn integer(n: f64) -> String {
+ format!("{}", n as u64)
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ #[test]
+ fn short_max_len() {
+ let mut float = 1.0;
+ while float < 999_999.9 {
+ let string = short(float);
+ println!("{}", string);
+ assert!(string.len() <= 6);
+ float *= 2.0;
+ }
+ }
+
+ #[test]
+ fn signed_short_max_len() {
+ let mut float = -1.0;
+ while float > -999_999.9 {
+ let string = signed_short(float);
+ println!("{}", string);
+ assert!(string.len() <= 7);
+ float *= 2.0;
+ }
+ }
+}
diff --git a/src/fs.rs b/src/fs.rs
new file mode 100755
index 0000000..f47508b
--- /dev/null
+++ b/src/fs.rs
@@ -0,0 +1,112 @@
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+use std::ffi::OsStr;
+use std::fs::{self, File};
+use std::io::Read;
+use std::path::Path;
+use walkdir::{DirEntry, WalkDir};
+
+use crate::error::{Error, Result};
+use crate::report::BenchmarkId;
+
+pub fn load<A, P: ?Sized>(path: &P) -> Result<A>
+where
+ A: DeserializeOwned,
+ P: AsRef<Path>,
+{
+ let path = path.as_ref();
+ let mut f = File::open(path).map_err(|inner| Error::AccessError {
+ inner,
+ path: path.to_owned(),
+ })?;
+ let mut string = String::new();
+ let _ = f.read_to_string(&mut string);
+ let result: A = serde_json::from_str(string.as_str()).map_err(|inner| Error::SerdeError {
+ inner,
+ path: path.to_owned(),
+ })?;
+
+ Ok(result)
+}
+
+pub fn is_dir<P>(path: &P) -> bool
+where
+ P: AsRef<Path>,
+{
+ let path: &Path = path.as_ref();
+ path.is_dir()
+}
+
+pub fn mkdirp<P>(path: &P) -> Result<()>
+where
+ P: AsRef<Path>,
+{
+ fs::create_dir_all(path.as_ref()).map_err(|inner| Error::AccessError {
+ inner,
+ path: path.as_ref().to_owned(),
+ })?;
+ Ok(())
+}
+
+pub fn cp(from: &Path, to: &Path) -> Result<()> {
+ fs::copy(from, to).map_err(|inner| Error::CopyError {
+ inner,
+ from: from.to_owned(),
+ to: to.to_owned(),
+ })?;
+ Ok(())
+}
+
+pub fn save<D, P>(data: &D, path: &P) -> Result<()>
+where
+ D: Serialize,
+ P: AsRef<Path>,
+{
+ let buf = serde_json::to_string(&data).map_err(|inner| Error::SerdeError {
+ path: path.as_ref().to_owned(),
+ inner,
+ })?;
+ save_string(&buf, path)
+}
+
+pub fn save_string<P>(data: &str, path: &P) -> Result<()>
+where
+ P: AsRef<Path>,
+{
+ use std::io::Write;
+
+ File::create(path)
+ .and_then(|mut f| f.write_all(data.as_bytes()))
+ .map_err(|inner| Error::AccessError {
+ inner,
+ path: path.as_ref().to_owned(),
+ })?;
+
+ Ok(())
+}
+
+pub fn list_existing_benchmarks<P>(directory: &P) -> Result<Vec<BenchmarkId>>
+where
+ P: AsRef<Path>,
+{
+ fn is_benchmark(entry: &DirEntry) -> bool {
+ // Look for benchmark.json files inside folders named "new" (because we want to ignore
+ // the baselines)
+ entry.file_name() == OsStr::new("benchmark.json")
+ && entry.path().parent().unwrap().file_name().unwrap() == OsStr::new("new")
+ }
+
+ let mut ids = vec![];
+
+ for entry in WalkDir::new(directory)
+ .into_iter()
+ // Ignore errors.
+ .filter_map(::std::result::Result::ok)
+ .filter(is_benchmark)
+ {
+ let id: BenchmarkId = load(entry.path())?;
+ ids.push(id);
+ }
+
+ Ok(ids)
+}
diff --git a/src/html/benchmark_report.html.tt b/src/html/benchmark_report.html.tt
new file mode 100755
index 0000000..5b0679e
--- /dev/null
+++ b/src/html/benchmark_report.html.tt
@@ -0,0 +1,308 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <title>{title} - Criterion.rs</title>
+ <style type="text/css">
+ body \{
+ font: 14px Helvetica Neue;
+ text-rendering: optimizelegibility;
+ }
+
+ .body \{
+ width: 960px;
+ margin: auto;
+ }
+
+ th \{
+ font-weight: 200
+ }
+
+ th,
+ td \{
+ padding-right: 3px;
+ padding-bottom: 3px;
+ }
+
+ a:link \{
+ color: #1F78B4;
+ text-decoration: none;
+ }
+
+ th.ci-bound \{
+ opacity: 0.6
+ }
+
+ td.ci-bound \{
+ opacity: 0.5
+ }
+
+ .stats \{
+ width: 80%;
+ margin: auto;
+ display: flex;
+ }
+
+ .additional_stats \{
+ flex: 0 0 60%
+ }
+
+ .additional_plots \{
+ flex: 1
+ }
+
+ h2 \{
+ font-size: 36px;
+ font-weight: 300;
+ }
+
+ h3 \{
+ font-size: 24px;
+ font-weight: 300;
+ }
+
+ #footer \{
+ height: 40px;
+ background: #888;
+ color: white;
+ font-size: larger;
+ font-weight: 300;
+ }
+
+ #footer a \{
+ color: white;
+ text-decoration: underline;
+ }
+
+ #footer p \{
+ text-align: center
+ }
+ </style>
+</head>
+
+<body>
+ <div class="body">
+ <h2>{title}</h2>
+ <div class="absolute">
+ <section class="plots">
+ <table width="100%">
+ <tbody>
+ <tr>
+ <td>
+ <a href="pdf.svg">
+ <img src="pdf_small.svg" alt="PDF of Slope" width="{thumbnail_width}" height="{thumbnail_height}" />
+ </a>
+ </td>
+ <td>
+ {{- if slope }}
+ <a href="regression.svg">
+ <img src="regression_small.svg" alt="Regression" width="{thumbnail_width}" height="{thumbnail_height}" />
+ </a>
+ {{- else }}
+ <a href="iteration_times.svg">
+ <img src="iteration_times_small.svg" alt="Iteration Times" width="{thumbnail_width}" height="{thumbnail_height}" />
+ </a>
+ {{- endif }}
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </section>
+ <section class="stats">
+ <div class="additional_stats">
+ <h4>Additional Statistics:</h4>
+ <table>
+ <thead>
+ <tr>
+ <th></th>
+ <th title="{confidence} confidence level" class="ci-bound">Lower bound</th>
+ <th>Estimate</th>
+ <th title="{confidence} confidence level" class="ci-bound">Upper bound</th>
+ </tr>
+ </thead>
+ <tbody>
+ {{- if slope }}
+ <tr>
+ <td>Slope</td>
+ <td class="ci-bound">{slope.lower}</td>
+ <td>{slope.point}</td>
+ <td class="ci-bound">{slope.upper}</td>
+ </tr>
+ {{- endif }}
+ {{- if throughput }}
+ <tr>
+ <td>Throughput</td>
+ <td class="ci-bound">{throughput.lower}</td>
+ <td>{throughput.point}</td>
+ <td class="ci-bound">{throughput.upper}</td>
+ </tr>
+ {{- endif }}
+ <tr>
+ <td>R&#xb2;</td>
+ <td class="ci-bound">{r2.lower}</td>
+ <td>{r2.point}</td>
+ <td class="ci-bound">{r2.upper}</td>
+ </tr>
+ <tr>
+ <td>Mean</td>
+ <td class="ci-bound">{mean.lower}</td>
+ <td>{mean.point}</td>
+ <td class="ci-bound">{mean.upper}</td>
+ </tr>
+ <tr>
+ <td title="Standard Deviation">Std. Dev.</td>
+ <td class="ci-bound">{std_dev.lower}</td>
+ <td>{std_dev.point}</td>
+ <td class="ci-bound">{std_dev.upper}</td>
+ </tr>
+ <tr>
+ <td>Median</td>
+ <td class="ci-bound">{median.lower}</td>
+ <td>{median.point}</td>
+ <td class="ci-bound">{median.upper}</td>
+ </tr>
+ <tr>
+ <td title="Median Absolute Deviation">MAD</td>
+ <td class="ci-bound">{mad.lower}</td>
+ <td>{mad.point}</td>
+ <td class="ci-bound">{mad.upper}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ {{- if additional_plots }}
+ <div class="additional_plots">
+ <h4>Additional Plots:</h4>
+ <ul>
+ {{for plot in additional_plots }}
+ <li>
+ <a href="{plot.url}">{plot.name}</a>
+ </li>
+ {{- endfor }}
+ </ul>
+ </div>
+ {{- endif }}
+ </section>
+ <section class="explanation">
+ <h4>Understanding this report:</h4>
+ <p>The plot on the left displays the average time per iteration for this benchmark. The shaded region
+ shows the estimated probabilty of an iteration taking a certain amount of time, while the line
+ shows the mean. Click on the plot for a larger view showing the outliers.</p>
+ {{- if slope }}
+ <p>The plot on the right shows the linear regression calculated from the measurements. Each point
+ represents a sample, though here it shows the total time for the sample rather than time per
+ iteration. The line is the line of best fit for these measurements.</p>
+ {{- else }}
+ <p>The plot on the right shows the average time per iteration for the samples. Each point
+ represents one sample.</p>
+ {{- endif }}
+ <p>See <a href="https://bheisler.github.io/criterion.rs/book/user_guide/command_line_output.html#additional-statistics">the
+ documentation</a> for more details on the additional statistics.</p>
+ </section>
+ </div>
+ {{- if comparison }}
+ <section class="plots">
+ <h3>Change Since Previous Benchmark</h3>
+ <div class="relative">
+ <table width="100%">
+ <tbody>
+ <tr>
+ <td>
+ <a href="both/pdf.svg">
+ <img src="relative_pdf_small.svg" alt="PDF Comparison" width="{thumbnail_width}"
+ height="{thumbnail_height}" />
+ </a>
+ </td>
+ <td>
+ {{- if slope }}
+ <a href="both/regression.svg">
+ <img src="relative_regression_small.svg" alt="Regression Comparison" width="{thumbnail_width}"
+ height="{thumbnail_height}" />
+ </a>
+ {{- else }}
+ <a href="both/iteration_times.svg">
+ <img src="relative_iteration_times_small.svg" alt="Iteration Time Comparison" width="{thumbnail_width}"
+ height="{thumbnail_height}" />
+ </a>
+ {{- endif }}
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </section>
+ <section class="stats">
+ <div class="additional_stats">
+ <h4>Additional Statistics:</h4>
+ <table>
+ <thead>
+ <tr>
+ <th></th>
+ <th title="{confidence} confidence level" class="ci-bound">Lower bound</th>
+ <th>Estimate</th>
+ <th title="{confidence} confidence level" class="ci-bound">Upper bound</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Change in time</td>
+ <td class="ci-bound">{comparison.change.lower}</td>
+ <td>{comparison.change.point}</td>
+ <td class="ci-bound">{comparison.change.upper}</td>
+ <td>(p = {comparison.p_value} {comparison.inequality}
+ {comparison.significance_level})</td>
+ </tr>
+ {{- if comparison.thrpt_change }}
+ <tr>
+ <td>Change in throughput</td>
+ <td class="ci-bound">{comparison.thrpt_change.lower}</td>
+ <td>{comparison.thrpt_change.point}</td>
+ <td class="ci-bound">{comparison.thrpt_change.upper}</td>
+ <td></td>
+ </tr>
+ {{- endif }}
+ </tbody>
+ </table>
+ {comparison.explanation}
+ </div>
+ {{- if comparison.additional_plots }}
+ <div class="additional_plots">
+ <h4>Additional Plots:</h4>
+ <ul>
+ {{ for plot in comparison.additional_plots }}
+ <li>
+ <a href="{plot.url}">{plot.name}</a>
+ </li>
+ {{- endfor }}
+ </ul>
+ </div>
+ {{- endif }}
+ </section>
+ <section class="explanation">
+ <h4>Understanding this report:</h4>
+ <p>The plot on the left shows the probability of the function taking a certain amount of time. The red
+ curve represents the saved measurements from the last time this benchmark was run, while the blue curve
+ shows the measurements from this run. The lines represent the mean time per iteration. Click on the
+ plot for a larger view.</p>
+ {{- if slope }}
+ <p>The plot on the right shows the two regressions. Again, the red line represents the previous measurement
+ while the blue line shows the current measurement.</p>
+ {{- else }}
+ <p>The plot on the right shows the iteration times for the two measurements. Again, the red dots represent
+ the previous measurement while the blue dots show the current measurement.</p>
+ {{- endif}}
+ <p>See <a href="https://bheisler.github.io/criterion.rs/book/user_guide/command_line_output.html#change">the
+ documentation</a> for more details on the additional statistics.</p>
+ </section>
+ {{- endif }}
+ </div>
+ <div id="footer">
+ <p>This report was generated by
+ <a href="https://github.com/bheisler/criterion.rs">Criterion.rs</a>, a statistics-driven benchmarking
+ library in Rust.</p>
+ </div>
+</body>
+
+</html> \ No newline at end of file
diff --git a/src/html/index.html.tt b/src/html/index.html.tt
new file mode 100755
index 0000000..7c307ed
--- /dev/null
+++ b/src/html/index.html.tt
@@ -0,0 +1,119 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <title>Index - Criterion.rs</title>
+ <style type="text/css">
+ body \{
+ font: 14px Helvetica Neue;
+ text-rendering: optimizelegibility;
+ }
+
+ .body \{
+ width: 960px;
+ margin: auto;
+ }
+
+ a:link \{
+ color: #1F78B4;
+ text-decoration: none;
+ }
+
+ h2 \{
+ font-size: 36px;
+ font-weight: 300;
+ }
+
+ h3 \{
+ font-size: 24px;
+ font-weight: 300;
+ }
+
+ #footer \{
+ height: 40px;
+ background: #888;
+ color: white;
+ font-size: larger;
+ font-weight: 300;
+ }
+
+ #footer a \{
+ color: white;
+ text-decoration: underline;
+ }
+
+ #footer p \{
+ text-align: center
+ }
+
+ table \{
+ border-collapse: collapse;
+ }
+
+ table,
+ th,
+ td \{
+ border: 1px solid #888;
+ }
+ </style>
+</head>
+
+<body>
+ <div class="body">
+ <h2>Criterion.rs Benchmark Index</h2>
+ See individual benchmark pages below for more details.
+ <ul>
+ {{- for group in groups }}
+ <li>{{ call report_link with group.group_report }}</li>
+ {{- if group.function_ids }}
+ {{- if group.values }}
+ {# Function ids and values #}
+ <ul>
+ <li>
+ <table>
+ <tr>
+ <th></th>
+ {{- for func in group.function_ids }}
+ <th>{{ call report_link with func }}</th>
+ {{- endfor }}
+ </tr>
+ {{- for row in group.individual_links }}
+ <tr>
+ <th>{{ call report_link with row.value }}</th>
+ {{- for bench in row.benchmarks }}
+ <td>{{ call report_link with bench }}</td>
+ {{- endfor }}
+ </tr>
+ {{- endfor }}
+ </table>
+ </li>
+ </ul>
+ {{- else }}
+ {# Function IDs but not values #}
+ <ul>
+ {{- for func in group.function_ids }}
+ <li>{{ call report_link with func }}</li>
+ {{- endfor }}
+ </ul>
+ {{- endif }}
+ {{- else }}
+ {{- if group.values }}
+ {# Values but not function ids #}
+ <ul>
+ {{- for val in group.values }}
+ <li>{{ call report_link with val }}</li>
+ {{- endfor }}
+ </ul>
+ {{- endif }}
+ {{- endif }}
+ {{- endfor }}
+ </ul>
+ </div>
+ <div id="footer">
+ <p>This report was generated by
+ <a href="https://github.com/bheisler/criterion.rs">Criterion.rs</a>, a statistics-driven benchmarking
+ library in Rust.</p>
+ </div>
+</body>
+</html> \ No newline at end of file
diff --git a/src/html/mod.rs b/src/html/mod.rs
new file mode 100755
index 0000000..02ff0de
--- /dev/null
+++ b/src/html/mod.rs
@@ -0,0 +1,837 @@
+use crate::report::{make_filename_safe, BenchmarkId, MeasurementData, Report, ReportContext};
+use crate::stats::bivariate::regression::Slope;
+
+use crate::estimate::Estimate;
+use crate::format;
+use crate::fs;
+use crate::measurement::ValueFormatter;
+use crate::plot::{PlotContext, PlotData, Plotter};
+use crate::SavedSample;
+use criterion_plot::Size;
+use serde::Serialize;
+use std::cell::RefCell;
+use std::cmp::Ordering;
+use std::collections::{BTreeSet, HashMap};
+use std::path::{Path, PathBuf};
+use tinytemplate::TinyTemplate;
+
+const THUMBNAIL_SIZE: Option<Size> = Some(Size(450, 300));
+
+fn debug_context<S: Serialize>(path: &Path, context: &S) {
+ if crate::debug_enabled() {
+ let mut context_path = PathBuf::from(path);
+ context_path.set_extension("json");
+ println!("Writing report context to {:?}", context_path);
+ let result = fs::save(context, &context_path);
+ if let Err(e) = result {
+ error!("Failed to write report context debug output: {}", e);
+ }
+ }
+}
+
+#[derive(Serialize)]
+struct Context {
+ title: String,
+ confidence: String,
+
+ thumbnail_width: usize,
+ thumbnail_height: usize,
+
+ slope: Option<ConfidenceInterval>,
+ r2: ConfidenceInterval,
+ mean: ConfidenceInterval,
+ std_dev: ConfidenceInterval,
+ median: ConfidenceInterval,
+ mad: ConfidenceInterval,
+ throughput: Option<ConfidenceInterval>,
+
+ additional_plots: Vec<Plot>,
+
+ comparison: Option<Comparison>,
+}
+
+#[derive(Serialize)]
+struct IndividualBenchmark {
+ name: String,
+ path: String,
+ regression_exists: bool,
+}
+impl IndividualBenchmark {
+ fn from_id(
+ output_directory: &Path,
+ path_prefix: &str,
+ id: &BenchmarkId,
+ ) -> IndividualBenchmark {
+ let mut regression_path = PathBuf::from(output_directory);
+ regression_path.push(id.as_directory_name());
+ regression_path.push("report");
+ regression_path.push("regression.svg");
+
+ IndividualBenchmark {
+ name: id.as_title().to_owned(),
+ path: format!("{}/{}", path_prefix, id.as_directory_name()),
+ regression_exists: regression_path.is_file(),
+ }
+ }
+}
+
+#[derive(Serialize)]
+struct SummaryContext {
+ group_id: String,
+
+ thumbnail_width: usize,
+ thumbnail_height: usize,
+
+ violin_plot: Option<String>,
+ line_chart: Option<String>,
+
+ benchmarks: Vec<IndividualBenchmark>,
+}
+
+#[derive(Serialize)]
+struct ConfidenceInterval {
+ lower: String,
+ upper: String,
+ point: String,
+}
+
+#[derive(Serialize)]
+struct Plot {
+ name: String,
+ url: String,
+}
+impl Plot {
+ fn new(name: &str, url: &str) -> Plot {
+ Plot {
+ name: name.to_owned(),
+ url: url.to_owned(),
+ }
+ }
+}
+
+#[derive(Serialize)]
+struct Comparison {
+ p_value: String,
+ inequality: String,
+ significance_level: String,
+ explanation: String,
+
+ change: ConfidenceInterval,
+ thrpt_change: Option<ConfidenceInterval>,
+ additional_plots: Vec<Plot>,
+}
+
+fn if_exists(output_directory: &Path, path: &Path) -> Option<String> {
+ let report_path = path.join("report/index.html");
+ if PathBuf::from(output_directory).join(&report_path).is_file() {
+ Some(report_path.to_string_lossy().to_string())
+ } else {
+ None
+ }
+}
+#[derive(Serialize, Debug)]
+struct ReportLink<'a> {
+ name: &'a str,
+ path: Option<String>,
+}
+impl<'a> ReportLink<'a> {
+ // TODO: Would be nice if I didn't have to keep making these components filename-safe.
+ fn group(output_directory: &Path, group_id: &'a str) -> ReportLink<'a> {
+ let path = PathBuf::from(make_filename_safe(group_id));
+
+ ReportLink {
+ name: group_id,
+ path: if_exists(output_directory, &path),
+ }
+ }
+
+ fn function(output_directory: &Path, group_id: &str, function_id: &'a str) -> ReportLink<'a> {
+ let mut path = PathBuf::from(make_filename_safe(group_id));
+ path.push(make_filename_safe(function_id));
+
+ ReportLink {
+ name: function_id,
+ path: if_exists(output_directory, &path),
+ }
+ }
+
+ fn value(output_directory: &Path, group_id: &str, value_str: &'a str) -> ReportLink<'a> {
+ let mut path = PathBuf::from(make_filename_safe(group_id));
+ path.push(make_filename_safe(value_str));
+
+ ReportLink {
+ name: value_str,
+ path: if_exists(output_directory, &path),
+ }
+ }
+
+ fn individual(output_directory: &Path, id: &'a BenchmarkId) -> ReportLink<'a> {
+ let path = PathBuf::from(id.as_directory_name());
+ ReportLink {
+ name: id.as_title(),
+ path: if_exists(output_directory, &path),
+ }
+ }
+}
+
+#[derive(Serialize)]
+struct BenchmarkValueGroup<'a> {
+ value: Option<ReportLink<'a>>,
+ benchmarks: Vec<ReportLink<'a>>,
+}
+
+#[derive(Serialize)]
+struct BenchmarkGroup<'a> {
+ group_report: ReportLink<'a>,
+
+ function_ids: Option<Vec<ReportLink<'a>>>,
+ values: Option<Vec<ReportLink<'a>>>,
+
+ individual_links: Vec<BenchmarkValueGroup<'a>>,
+}
+impl<'a> BenchmarkGroup<'a> {
+ fn new(output_directory: &Path, ids: &[&'a BenchmarkId]) -> BenchmarkGroup<'a> {
+ let group_id = &ids[0].group_id;
+ let group_report = ReportLink::group(output_directory, group_id);
+
+ let mut function_ids = Vec::with_capacity(ids.len());
+ let mut values = Vec::with_capacity(ids.len());
+ let mut individual_links = HashMap::with_capacity(ids.len());
+
+ for id in ids.iter() {
+ let function_id = id.function_id.as_ref().map(String::as_str);
+ let value = id.value_str.as_ref().map(String::as_str);
+
+ let individual_link = ReportLink::individual(output_directory, id);
+
+ function_ids.push(function_id);
+ values.push(value);
+
+ individual_links.insert((function_id, value), individual_link);
+ }
+
+ fn parse_opt(os: &Option<&str>) -> Option<f64> {
+ os.and_then(|s| s.parse::<f64>().ok())
+ }
+
+ // If all of the value strings can be parsed into a number, sort/dedupe
+ // numerically. Otherwise sort lexicographically.
+ if values.iter().all(|os| parse_opt(os).is_some()) {
+ values.sort_unstable_by(|v1, v2| {
+ let num1 = parse_opt(&v1);
+ let num2 = parse_opt(&v2);
+
+ num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
+ });
+ values.dedup_by_key(|os| parse_opt(&os).unwrap());
+ } else {
+ values.sort_unstable();
+ values.dedup();
+ }
+
+ // Sort and dedupe functions by name.
+ function_ids.sort_unstable();
+ function_ids.dedup();
+
+ let mut value_groups = Vec::with_capacity(values.len());
+ for value in values.iter() {
+ let row = function_ids
+ .iter()
+ .filter_map(|f| individual_links.remove(&(*f, *value)))
+ .collect::<Vec<_>>();
+ value_groups.push(BenchmarkValueGroup {
+ value: value.map(|s| ReportLink::value(output_directory, group_id, s)),
+ benchmarks: row,
+ });
+ }
+
+ let function_ids = function_ids
+ .into_iter()
+ .map(|os| os.map(|s| ReportLink::function(output_directory, group_id, s)))
+ .collect::<Option<Vec<_>>>();
+ let values = values
+ .into_iter()
+ .map(|os| os.map(|s| ReportLink::value(output_directory, group_id, s)))
+ .collect::<Option<Vec<_>>>();
+
+ BenchmarkGroup {
+ group_report,
+ function_ids,
+ values,
+ individual_links: value_groups,
+ }
+ }
+}
+
+#[derive(Serialize)]
+struct IndexContext<'a> {
+ groups: Vec<BenchmarkGroup<'a>>,
+}
+
+pub struct Html {
+ templates: TinyTemplate<'static>,
+ plotter: RefCell<Box<dyn Plotter>>,
+}
+impl Html {
+ pub(crate) fn new(plotter: Box<dyn Plotter>) -> Html {
+ let mut templates = TinyTemplate::new();
+ templates
+ .add_template("report_link", include_str!("report_link.html.tt"))
+ .expect("Unable to parse report_link template.");
+ templates
+ .add_template("index", include_str!("index.html.tt"))
+ .expect("Unable to parse index template.");
+ templates
+ .add_template("benchmark_report", include_str!("benchmark_report.html.tt"))
+ .expect("Unable to parse benchmark_report template");
+ templates
+ .add_template("summary_report", include_str!("summary_report.html.tt"))
+ .expect("Unable to parse summary_report template");
+
+ let plotter = RefCell::new(plotter);
+ Html { templates, plotter }
+ }
+}
+impl Report for Html {
+ fn measurement_complete(
+ &self,
+ id: &BenchmarkId,
+ report_context: &ReportContext,
+ measurements: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) {
+ try_else_return!({
+ let mut report_dir = report_context.output_directory.clone();
+ report_dir.push(id.as_directory_name());
+ report_dir.push("report");
+ fs::mkdirp(&report_dir)
+ });
+
+ let typical_estimate = &measurements.absolute_estimates.typical();
+
+ let time_interval = |est: &Estimate| -> ConfidenceInterval {
+ ConfidenceInterval {
+ lower: formatter.format_value(est.confidence_interval.lower_bound),
+ point: formatter.format_value(est.point_estimate),
+ upper: formatter.format_value(est.confidence_interval.upper_bound),
+ }
+ };
+
+ let data = measurements.data;
+
+ elapsed! {
+ "Generating plots",
+ self.generate_plots(id, report_context, formatter, measurements)
+ }
+
+ let mut additional_plots = vec![
+ Plot::new("Typical", "typical.svg"),
+ Plot::new("Mean", "mean.svg"),
+ Plot::new("Std. Dev.", "SD.svg"),
+ Plot::new("Median", "median.svg"),
+ Plot::new("MAD", "MAD.svg"),
+ ];
+ if measurements.absolute_estimates.slope.is_some() {
+ additional_plots.push(Plot::new("Slope", "slope.svg"));
+ }
+
+ let throughput = measurements
+ .throughput
+ .as_ref()
+ .map(|thr| ConfidenceInterval {
+ lower: formatter
+ .format_throughput(thr, typical_estimate.confidence_interval.upper_bound),
+ upper: formatter
+ .format_throughput(thr, typical_estimate.confidence_interval.lower_bound),
+ point: formatter.format_throughput(thr, typical_estimate.point_estimate),
+ });
+
+ let context = Context {
+ title: id.as_title().to_owned(),
+ confidence: format!(
+ "{:.2}",
+ typical_estimate.confidence_interval.confidence_level
+ ),
+
+ thumbnail_width: THUMBNAIL_SIZE.unwrap().0,
+ thumbnail_height: THUMBNAIL_SIZE.unwrap().1,
+
+ slope: measurements
+ .absolute_estimates
+ .slope
+ .as_ref()
+ .map(time_interval),
+ mean: time_interval(&measurements.absolute_estimates.mean),
+ median: time_interval(&measurements.absolute_estimates.median),
+ mad: time_interval(&measurements.absolute_estimates.median_abs_dev),
+ std_dev: time_interval(&measurements.absolute_estimates.std_dev),
+ throughput,
+
+ r2: ConfidenceInterval {
+ lower: format!(
+ "{:0.7}",
+ Slope(typical_estimate.confidence_interval.lower_bound).r_squared(&data)
+ ),
+ upper: format!(
+ "{:0.7}",
+ Slope(typical_estimate.confidence_interval.upper_bound).r_squared(&data)
+ ),
+ point: format!(
+ "{:0.7}",
+ Slope(typical_estimate.point_estimate).r_squared(&data)
+ ),
+ },
+
+ additional_plots,
+
+ comparison: self.comparison(measurements),
+ };
+
+ let mut report_path = report_context.output_directory.clone();
+ report_path.push(id.as_directory_name());
+ report_path.push("report");
+ report_path.push("index.html");
+ debug_context(&report_path, &context);
+
+ let text = self
+ .templates
+ .render("benchmark_report", &context)
+ .expect("Failed to render benchmark report template");
+ try_else_return!(fs::save_string(&text, &report_path));
+ }
+
+ fn summarize(
+ &self,
+ context: &ReportContext,
+ all_ids: &[BenchmarkId],
+ formatter: &dyn ValueFormatter,
+ ) {
+ let all_ids = all_ids
+ .iter()
+ .filter(|id| {
+ let id_dir = context.output_directory.join(id.as_directory_name());
+ fs::is_dir(&id_dir)
+ })
+ .collect::<Vec<_>>();
+ if all_ids.is_empty() {
+ return;
+ }
+
+ let group_id = all_ids[0].group_id.clone();
+
+ let data = self.load_summary_data(&context.output_directory, &all_ids);
+
+ let mut function_ids = BTreeSet::new();
+ let mut value_strs = Vec::with_capacity(all_ids.len());
+ for id in all_ids {
+ if let Some(ref function_id) = id.function_id {
+ function_ids.insert(function_id);
+ }
+ if let Some(ref value_str) = id.value_str {
+ value_strs.push(value_str);
+ }
+ }
+
+ fn try_parse(s: &str) -> Option<f64> {
+ s.parse::<f64>().ok()
+ }
+
+ // If all of the value strings can be parsed into a number, sort/dedupe
+ // numerically. Otherwise sort lexicographically.
+ if value_strs.iter().all(|os| try_parse(&*os).is_some()) {
+ value_strs.sort_unstable_by(|v1, v2| {
+ let num1 = try_parse(&v1);
+ let num2 = try_parse(&v2);
+
+ num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
+ });
+ value_strs.dedup_by_key(|os| try_parse(&os).unwrap());
+ } else {
+ value_strs.sort_unstable();
+ value_strs.dedup();
+ }
+
+ for function_id in function_ids {
+ let samples_with_function: Vec<_> = data
+ .iter()
+ .by_ref()
+ .filter(|&&(ref id, _)| id.function_id.as_ref() == Some(&function_id))
+ .collect();
+
+ if samples_with_function.len() > 1 {
+ let subgroup_id =
+ BenchmarkId::new(group_id.clone(), Some(function_id.clone()), None, None);
+
+ self.generate_summary(
+ &subgroup_id,
+ &*samples_with_function,
+ context,
+ formatter,
+ false,
+ );
+ }
+ }
+
+ for value_str in value_strs {
+ let samples_with_value: Vec<_> = data
+ .iter()
+ .by_ref()
+ .filter(|&&(ref id, _)| id.value_str.as_ref() == Some(&value_str))
+ .collect();
+
+ if samples_with_value.len() > 1 {
+ let subgroup_id =
+ BenchmarkId::new(group_id.clone(), None, Some(value_str.clone()), None);
+
+ self.generate_summary(
+ &subgroup_id,
+ &*samples_with_value,
+ context,
+ formatter,
+ false,
+ );
+ }
+ }
+
+ let mut all_data = data.iter().by_ref().collect::<Vec<_>>();
+ // First sort the ids/data by value.
+ // If all of the value strings can be parsed into a number, sort/dedupe
+ // numerically. Otherwise sort lexicographically.
+ let all_values_numeric = all_data.iter().all(|(ref id, _)| {
+ id.value_str
+ .as_ref()
+ .map(String::as_str)
+ .and_then(try_parse)
+ .is_some()
+ });
+ if all_values_numeric {
+ all_data.sort_unstable_by(|(a, _), (b, _)| {
+ let num1 = a.value_str.as_ref().map(String::as_str).and_then(try_parse);
+ let num2 = b.value_str.as_ref().map(String::as_str).and_then(try_parse);
+
+ num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
+ });
+ } else {
+ all_data.sort_unstable_by_key(|(id, _)| id.value_str.as_ref());
+ }
+ // Next, sort the ids/data by function name. This results in a sorting priority of
+ // function name, then value. This one has to be a stable sort.
+ all_data.sort_by_key(|(id, _)| id.function_id.as_ref());
+
+ self.generate_summary(
+ &BenchmarkId::new(group_id, None, None, None),
+ &*(all_data),
+ context,
+ formatter,
+ true,
+ );
+ self.plotter.borrow_mut().wait();
+ }
+
+ fn final_summary(&self, report_context: &ReportContext) {
+ let output_directory = &report_context.output_directory;
+ if !fs::is_dir(&output_directory) {
+ return;
+ }
+
+ let mut found_ids = try_else_return!(fs::list_existing_benchmarks(&output_directory));
+ found_ids.sort_unstable_by_key(|id| id.id().to_owned());
+
+ // Group IDs by group id
+ let mut id_groups: HashMap<&str, Vec<&BenchmarkId>> = HashMap::new();
+ for id in found_ids.iter() {
+ id_groups
+ .entry(&id.group_id)
+ .or_insert_with(Vec::new)
+ .push(id);
+ }
+
+ let mut groups = id_groups
+ .into_iter()
+ .map(|(_, group)| BenchmarkGroup::new(output_directory, &group))
+ .collect::<Vec<BenchmarkGroup<'_>>>();
+ groups.sort_unstable_by_key(|g| g.group_report.name);
+
+ try_else_return!(fs::mkdirp(&output_directory.join("report")));
+
+ let report_path = output_directory.join("report").join("index.html");
+
+ let context = IndexContext { groups };
+
+ debug_context(&report_path, &context);
+
+ let text = self
+ .templates
+ .render("index", &context)
+ .expect("Failed to render index template");
+ try_else_return!(fs::save_string(&text, &report_path,));
+ }
+}
+impl Html {
+ fn comparison(&self, measurements: &MeasurementData<'_>) -> Option<Comparison> {
+ if let Some(ref comp) = measurements.comparison {
+ let different_mean = comp.p_value < comp.significance_threshold;
+ let mean_est = &comp.relative_estimates.mean;
+ let explanation_str: String;
+
+ if !different_mean {
+ explanation_str = "No change in performance detected.".to_owned();
+ } else {
+ let comparison = compare_to_threshold(&mean_est, comp.noise_threshold);
+ match comparison {
+ ComparisonResult::Improved => {
+ explanation_str = "Performance has improved.".to_owned();
+ }
+ ComparisonResult::Regressed => {
+ explanation_str = "Performance has regressed.".to_owned();
+ }
+ ComparisonResult::NonSignificant => {
+ explanation_str = "Change within noise threshold.".to_owned();
+ }
+ }
+ }
+
+ let comp = Comparison {
+ p_value: format!("{:.2}", comp.p_value),
+ inequality: (if different_mean { "<" } else { ">" }).to_owned(),
+ significance_level: format!("{:.2}", comp.significance_threshold),
+ explanation: explanation_str,
+
+ change: ConfidenceInterval {
+ point: format::change(mean_est.point_estimate, true),
+ lower: format::change(mean_est.confidence_interval.lower_bound, true),
+ upper: format::change(mean_est.confidence_interval.upper_bound, true),
+ },
+
+ thrpt_change: measurements.throughput.as_ref().map(|_| {
+ let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
+ ConfidenceInterval {
+ point: format::change(to_thrpt_estimate(mean_est.point_estimate), true),
+ lower: format::change(
+ to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
+ true,
+ ),
+ upper: format::change(
+ to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
+ true,
+ ),
+ }
+ }),
+
+ additional_plots: vec![
+ Plot::new("Change in mean", "change/mean.svg"),
+ Plot::new("Change in median", "change/median.svg"),
+ Plot::new("T-Test", "change/t-test.svg"),
+ ],
+ };
+ Some(comp)
+ } else {
+ None
+ }
+ }
+
+ fn generate_plots(
+ &self,
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ ) {
+ let plot_ctx = PlotContext {
+ id,
+ context,
+ size: None,
+ is_thumbnail: false,
+ };
+
+ let plot_data = PlotData {
+ measurements,
+ formatter,
+ comparison: None,
+ };
+
+ let plot_ctx_small = plot_ctx.thumbnail(true).size(THUMBNAIL_SIZE);
+
+ self.plotter.borrow_mut().pdf(plot_ctx, plot_data);
+ self.plotter.borrow_mut().pdf(plot_ctx_small, plot_data);
+ if measurements.absolute_estimates.slope.is_some() {
+ self.plotter.borrow_mut().regression(plot_ctx, plot_data);
+ self.plotter
+ .borrow_mut()
+ .regression(plot_ctx_small, plot_data);
+ } else {
+ self.plotter
+ .borrow_mut()
+ .iteration_times(plot_ctx, plot_data);
+ self.plotter
+ .borrow_mut()
+ .iteration_times(plot_ctx_small, plot_data);
+ }
+
+ self.plotter
+ .borrow_mut()
+ .abs_distributions(plot_ctx, plot_data);
+
+ if let Some(ref comp) = measurements.comparison {
+ try_else_return!({
+ let mut change_dir = context.output_directory.clone();
+ change_dir.push(id.as_directory_name());
+ change_dir.push("report");
+ change_dir.push("change");
+ fs::mkdirp(&change_dir)
+ });
+
+ try_else_return!({
+ let mut both_dir = context.output_directory.clone();
+ both_dir.push(id.as_directory_name());
+ both_dir.push("report");
+ both_dir.push("both");
+ fs::mkdirp(&both_dir)
+ });
+
+ let comp_data = plot_data.comparison(&comp);
+
+ self.plotter.borrow_mut().pdf(plot_ctx, comp_data);
+ self.plotter.borrow_mut().pdf(plot_ctx_small, comp_data);
+ if measurements.absolute_estimates.slope.is_some()
+ && comp.base_estimates.slope.is_some()
+ {
+ self.plotter.borrow_mut().regression(plot_ctx, comp_data);
+ self.plotter
+ .borrow_mut()
+ .regression(plot_ctx_small, comp_data);
+ } else {
+ self.plotter
+ .borrow_mut()
+ .iteration_times(plot_ctx, comp_data);
+ self.plotter
+ .borrow_mut()
+ .iteration_times(plot_ctx_small, comp_data);
+ }
+ self.plotter.borrow_mut().t_test(plot_ctx, comp_data);
+ self.plotter
+ .borrow_mut()
+ .rel_distributions(plot_ctx, comp_data);
+ }
+
+ self.plotter.borrow_mut().wait();
+ }
+
+ fn load_summary_data<'a>(
+ &self,
+ output_directory: &Path,
+ all_ids: &[&'a BenchmarkId],
+ ) -> Vec<(&'a BenchmarkId, Vec<f64>)> {
+ all_ids
+ .iter()
+ .filter_map(|id| {
+ let entry = output_directory.join(id.as_directory_name()).join("new");
+
+ let SavedSample { iters, times, .. } =
+ try_else_return!(fs::load(&entry.join("sample.json")), || None);
+ let avg_times = iters
+ .into_iter()
+ .zip(times.into_iter())
+ .map(|(iters, time)| time / iters)
+ .collect::<Vec<_>>();
+
+ Some((*id, avg_times))
+ })
+ .collect::<Vec<_>>()
+ }
+
+ fn generate_summary(
+ &self,
+ id: &BenchmarkId,
+ data: &[&(&BenchmarkId, Vec<f64>)],
+ report_context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ full_summary: bool,
+ ) {
+ let plot_ctx = PlotContext {
+ id,
+ context: report_context,
+ size: None,
+ is_thumbnail: false,
+ };
+
+ try_else_return!(
+ {
+ let mut report_dir = report_context.output_directory.clone();
+ report_dir.push(id.as_directory_name());
+ report_dir.push("report");
+ fs::mkdirp(&report_dir)
+ },
+ || {}
+ );
+
+ self.plotter.borrow_mut().violin(plot_ctx, formatter, data);
+
+ let value_types: Vec<_> = data.iter().map(|&&(ref id, _)| id.value_type()).collect();
+ let mut line_path = None;
+
+ if value_types.iter().all(|x| x == &value_types[0]) {
+ if let Some(value_type) = value_types[0] {
+ let values: Vec<_> = data.iter().map(|&&(ref id, _)| id.as_number()).collect();
+ if values.iter().any(|x| x != &values[0]) {
+ self.plotter
+ .borrow_mut()
+ .line_comparison(plot_ctx, formatter, data, value_type);
+ line_path = Some(plot_ctx.line_comparison_path());
+ }
+ }
+ }
+
+ let path_prefix = if full_summary { "../.." } else { "../../.." };
+ let benchmarks = data
+ .iter()
+ .map(|&&(ref id, _)| {
+ IndividualBenchmark::from_id(&report_context.output_directory, path_prefix, id)
+ })
+ .collect();
+
+ let context = SummaryContext {
+ group_id: id.as_title().to_owned(),
+
+ thumbnail_width: THUMBNAIL_SIZE.unwrap().0,
+ thumbnail_height: THUMBNAIL_SIZE.unwrap().1,
+
+ violin_plot: Some(plot_ctx.violin_path().to_string_lossy().into_owned()),
+ line_chart: line_path.map(|p| p.to_string_lossy().into_owned()),
+
+ benchmarks,
+ };
+
+ let mut report_path = report_context.output_directory.clone();
+ report_path.push(id.as_directory_name());
+ report_path.push("report");
+ report_path.push("index.html");
+ debug_context(&report_path, &context);
+
+ let text = self
+ .templates
+ .render("summary_report", &context)
+ .expect("Failed to render summary report template");
+ try_else_return!(fs::save_string(&text, &report_path,), || {});
+ }
+}
+
+enum ComparisonResult {
+ Improved,
+ Regressed,
+ NonSignificant,
+}
+
+fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
+ let ci = &estimate.confidence_interval;
+ let lb = ci.lower_bound;
+ let ub = ci.upper_bound;
+
+ if lb < -noise && ub < -noise {
+ ComparisonResult::Improved
+ } else if lb > noise && ub > noise {
+ ComparisonResult::Regressed
+ } else {
+ ComparisonResult::NonSignificant
+ }
+}
diff --git a/src/html/report_link.html.tt b/src/html/report_link.html.tt
new file mode 100755
index 0000000..6013114
--- /dev/null
+++ b/src/html/report_link.html.tt
@@ -0,0 +1,5 @@
+{{ if path -}}
+<a href="../{path}">{name}</a>
+{{- else -}}
+{name}
+{{- endif}} \ No newline at end of file
diff --git a/src/html/summary_report.html.tt b/src/html/summary_report.html.tt
new file mode 100755
index 0000000..4f36f62
--- /dev/null
+++ b/src/html/summary_report.html.tt
@@ -0,0 +1,109 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <title>{group_id} Summary - Criterion.rs</title>
+ <style type="text/css">
+ body \{
+ font: 14px Helvetica Neue;
+ text-rendering: optimizelegibility;
+ }
+
+ .body \{
+ width: 960px;
+ margin: auto;
+ }
+
+ a:link \{
+ color: #1F78B4;
+ text-decoration: none;
+ }
+
+ h2 \{
+ font-size: 36px;
+ font-weight: 300;
+ }
+
+ h3 \{
+ font-size: 24px;
+ font-weight: 300;
+ }
+
+ #footer \{
+ height: 40px;
+ background: #888;
+ color: white;
+ font-size: larger;
+ font-weight: 300;
+ }
+
+ #footer a \{
+ color: white;
+ text-decoration: underline;
+ }
+
+ #footer p \{
+ text-align: center
+ }
+ </style>
+</head>
+
+<body>
+ <div class="body">
+ <h2>{group_id}</h2>
+ {{- if violin_plot }}
+ <h3>Violin Plot</h3>
+ <a href="violin.svg">
+ <img src="violin.svg" alt="Violin Plot" />
+ </a>
+ <p>This chart shows the relationship between function/parameter and iteration time. The thickness of the shaded
+ region indicates the probability that a measurement of the given function/parameter would take a particular
+ length of time.</p>
+ {{- endif }}
+ {{- if line_chart }}
+ <h3>Line Chart</h3>
+ <img src="lines.svg" alt="Line Chart" />
+ <p>This chart shows the mean measured time for each function as the input (or the size of the input) increases.</p>
+ {{- endif }}
+ {{- for bench in benchmarks }}
+ <section class="plots">
+ <a href="{bench.path}/report/index.html">
+ <h4>{bench.name}</h4>
+ </a>
+ <table width="100%">
+ <tbody>
+ <tr>
+ <td>
+ <a href="{bench.path}/report/pdf.svg">
+ <img src="{bench.path}/report/pdf_small.svg" alt="PDF of Slope" width="{thumbnail_width}"
+ height="{thumbnail_height}" />
+ </a>
+ </td>
+ <td>
+ {{- if bench.regression_exists }}
+ <a href="{bench.path}/report/regression.svg">
+ <img src="{bench.path}/report/regression_small.svg" alt="Regression" width="{thumbnail_width}"
+ height="{thumbnail_height}" />
+ </a>
+ {{- else }}
+ <a href="{bench.path}/report/iteration_times.svg">
+ <img src="{bench.path}/report/iteration_times_small.svg" alt="Iteration Times" width="{thumbnail_width}"
+ height="{thumbnail_height}" />
+ </a>
+ {{- endif }}
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </section>
+ {{- endfor }}
+ </div>
+ <div id="footer">
+ <p>This report was generated by
+ <a href="https://github.com/bheisler/criterion.rs">Criterion.rs</a>, a statistics-driven benchmarking
+ library in Rust.</p>
+ </div>
+</body>
+
+</html> \ No newline at end of file
diff --git a/src/kde.rs b/src/kde.rs
new file mode 100755
index 0000000..8812142
--- /dev/null
+++ b/src/kde.rs
@@ -0,0 +1,41 @@
+use crate::stats::univariate::kde::kernel::Gaussian;
+use crate::stats::univariate::kde::{Bandwidth, Kde};
+use crate::stats::univariate::Sample;
+
+pub fn sweep(
+ sample: &Sample<f64>,
+ npoints: usize,
+ range: Option<(f64, f64)>,
+) -> (Box<[f64]>, Box<[f64]>) {
+ let (xs, ys, _) = sweep_and_estimate(sample, npoints, range, sample[0]);
+ (xs, ys)
+}
+
+pub fn sweep_and_estimate(
+ sample: &Sample<f64>,
+ npoints: usize,
+ range: Option<(f64, f64)>,
+ point_to_estimate: f64,
+) -> (Box<[f64]>, Box<[f64]>, f64) {
+ let x_min = sample.min();
+ let x_max = sample.max();
+
+ let kde = Kde::new(sample, Gaussian, Bandwidth::Silverman);
+ let h = kde.bandwidth();
+
+ let (start, end) = match range {
+ Some((start, end)) => (start, end),
+ None => (x_min - 3. * h, x_max + 3. * h),
+ };
+
+ let mut xs: Vec<f64> = Vec::with_capacity(npoints);
+ let step_size = (end - start) / (npoints - 1) as f64;
+ for n in 0..npoints {
+ xs.push(start + (step_size * n as f64));
+ }
+
+ let ys = kde.map(&xs);
+ let point_estimate = kde.estimate(point_to_estimate);
+
+ (xs.into_boxed_slice(), ys, point_estimate)
+}
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100755
index 0000000..0fd273c
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1,1899 @@
+//! A statistics-driven micro-benchmarking library written in Rust.
+//!
+//! This crate is a microbenchmarking library which aims to provide strong
+//! statistical confidence in detecting and estimating the size of performance
+//! improvements and regressions, while also being easy to use.
+//!
+//! See
+//! [the user guide](https://bheisler.github.io/criterion.rs/book/index.html)
+//! for examples as well as details on the measurement and analysis process,
+//! and the output.
+//!
+//! ## Features:
+//! * Collects detailed statistics, providing strong confidence that changes
+//! to performance are real, not measurement noise
+//! * Produces detailed charts, providing thorough understanding of your code's
+//! performance behavior.
+
+#![warn(missing_docs)]
+#![warn(bare_trait_objects)]
+#![cfg_attr(feature = "real_blackbox", feature(test))]
+#![cfg_attr(
+ feature = "cargo-clippy",
+ allow(
+ clippy::just_underscores_and_digits, // Used in the stats code
+ clippy::transmute_ptr_to_ptr, // Used in the stats code
+ clippy::option_as_ref_deref, // Remove when MSRV bumped above 1.40
+ )
+)]
+
+#[cfg(test)]
+extern crate approx;
+
+#[cfg(test)]
+extern crate quickcheck;
+
+use clap::value_t;
+use regex::Regex;
+
+#[macro_use]
+extern crate lazy_static;
+
+#[cfg(feature = "real_blackbox")]
+extern crate test;
+
+#[macro_use]
+extern crate serde_derive;
+
+// Needs to be declared before other modules
+// in order to be usable there.
+#[macro_use]
+mod macros_private;
+#[macro_use]
+mod analysis;
+mod benchmark;
+#[macro_use]
+mod benchmark_group;
+mod connection;
+mod csv_report;
+mod error;
+mod estimate;
+mod format;
+mod fs;
+mod html;
+mod kde;
+mod macros;
+pub mod measurement;
+mod plot;
+pub mod profiler;
+mod report;
+mod routine;
+mod stats;
+
+use std::cell::RefCell;
+use std::collections::HashSet;
+use std::default::Default;
+use std::fmt;
+use std::iter::IntoIterator;
+use std::marker::PhantomData;
+use std::net::TcpStream;
+use std::path::{Path, PathBuf};
+use std::sync::{Mutex, MutexGuard};
+use std::time::Duration;
+use std::time::Instant;
+
+use criterion_plot::{Version, VersionError};
+
+use crate::benchmark::BenchmarkConfig;
+use crate::benchmark::NamedRoutine;
+use crate::connection::Connection;
+use crate::connection::OutgoingMessage;
+use crate::csv_report::FileCsvReport;
+use crate::html::Html;
+use crate::measurement::{Measurement, WallTime};
+use crate::plot::{Gnuplot, Plotter, PlottersBackend};
+use crate::profiler::{ExternalProfiler, Profiler};
+use crate::report::{BencherReport, CliReport, Report, ReportContext, Reports};
+use crate::routine::Function;
+
+pub use crate::benchmark::{Benchmark, BenchmarkDefinition, ParameterizedBenchmark};
+pub use crate::benchmark_group::{BenchmarkGroup, BenchmarkId};
+
+lazy_static! {
+ static ref DEBUG_ENABLED: bool = std::env::var_os("CRITERION_DEBUG").is_some();
+ static ref GNUPLOT_VERSION: Result<Version, VersionError> = criterion_plot::version();
+ static ref DEFAULT_PLOTTING_BACKEND: PlottingBackend = {
+ match &*GNUPLOT_VERSION {
+ Ok(_) => PlottingBackend::Gnuplot,
+ Err(e) => {
+ match e {
+ VersionError::Exec(_) => println!("Gnuplot not found, using plotters backend"),
+ e => println!(
+ "Gnuplot not found or not usable, using plotters backend\n{}",
+ e
+ ),
+ };
+ PlottingBackend::Plotters
+ }
+ }
+ };
+ static ref CARGO_CRITERION_CONNECTION: Option<Mutex<Connection>> = {
+ match std::env::var("CARGO_CRITERION_PORT") {
+ Ok(port_str) => {
+ let port: u16 = port_str.parse().ok()?;
+ let stream = TcpStream::connect(("localhost", port)).ok()?;
+ Some(Mutex::new(Connection::new(stream).ok()?))
+ }
+ Err(_) => None,
+ }
+ };
+}
+
+fn debug_enabled() -> bool {
+ *DEBUG_ENABLED
+}
+
+/// A function that is opaque to the optimizer, used to prevent the compiler from
+/// optimizing away computations in a benchmark.
+///
+/// This variant is backed by the (unstable) test::black_box function.
+#[cfg(feature = "real_blackbox")]
+pub fn black_box<T>(dummy: T) -> T {
+ test::black_box(dummy)
+}
+
+/// A function that is opaque to the optimizer, used to prevent the compiler from
+/// optimizing away computations in a benchmark.
+///
+/// This variant is stable-compatible, but it may cause some performance overhead
+/// or fail to prevent code from being eliminated.
+#[cfg(not(feature = "real_blackbox"))]
+pub fn black_box<T>(dummy: T) -> T {
+ unsafe {
+ let ret = std::ptr::read_volatile(&dummy);
+ std::mem::forget(dummy);
+ ret
+ }
+}
+
+/// Representing a function to benchmark together with a name of that function.
+/// Used together with `bench_functions` to represent one out of multiple functions
+/// under benchmark.
+#[doc(hidden)]
+pub struct Fun<I: fmt::Debug, M: Measurement + 'static = WallTime> {
+ f: NamedRoutine<I, M>,
+ _phantom: PhantomData<M>,
+}
+
+impl<I, M: Measurement> Fun<I, M>
+where
+ I: fmt::Debug + 'static,
+{
+ /// Create a new `Fun` given a name and a closure
+ pub fn new<F>(name: &str, f: F) -> Fun<I, M>
+ where
+ F: FnMut(&mut Bencher<'_, M>, &I) + 'static,
+ {
+ let routine = NamedRoutine {
+ id: name.to_owned(),
+ f: Box::new(RefCell::new(Function::new(f))),
+ };
+
+ Fun {
+ f: routine,
+ _phantom: PhantomData,
+ }
+ }
+}
+
+/// Argument to [`Bencher::iter_batched`](struct.Bencher.html#method.iter_batched) and
+/// [`Bencher::iter_batched_ref`](struct.Bencher.html#method.iter_batched_ref) which controls the
+/// batch size.
+///
+/// Generally speaking, almost all benchmarks should use `SmallInput`. If the input or the result
+/// of the benchmark routine is large enough that `SmallInput` causes out-of-memory errors,
+/// `LargeInput` can be used to reduce memory usage at the cost of increasing the measurement
+/// overhead. If the input or the result is extremely large (or if it holds some
+/// limited external resource like a file handle), `PerIteration` will set the number of iterations
+/// per batch to exactly one. `PerIteration` can increase the measurement overhead substantially
+/// and should be avoided wherever possible.
+///
+/// Each value lists an estimate of the measurement overhead. This is intended as a rough guide
+/// to assist in choosing an option, it should not be relied upon. In particular, it is not valid
+/// to subtract the listed overhead from the measurement and assume that the result represents the
+/// true runtime of a function. The actual measurement overhead for your specific benchmark depends
+/// on the details of the function you're benchmarking and the hardware and operating
+/// system running the benchmark.
+///
+/// With that said, if the runtime of your function is small relative to the measurement overhead
+/// it will be difficult to take accurate measurements. In this situation, the best option is to use
+/// [`Bencher::iter`](struct.Bencher.html#method.iter) which has next-to-zero measurement overhead.
+#[derive(Debug, Eq, PartialEq, Copy, Hash, Clone)]
+pub enum BatchSize {
+ /// `SmallInput` indicates that the input to the benchmark routine (the value returned from
+ /// the setup routine) is small enough that millions of values can be safely held in memory.
+ /// Always prefer `SmallInput` unless the benchmark is using too much memory.
+ ///
+ /// In testing, the maximum measurement overhead from benchmarking with `SmallInput` is on the
+ /// order of 500 picoseconds. This is presented as a rough guide; your results may vary.
+ SmallInput,
+
+ /// `LargeInput` indicates that the input to the benchmark routine or the value returned from
+ /// that routine is large. This will reduce the memory usage but increase the measurement
+ /// overhead.
+ ///
+ /// In testing, the maximum measurement overhead from benchmarking with `LargeInput` is on the
+ /// order of 750 picoseconds. This is presented as a rough guide; your results may vary.
+ LargeInput,
+
+ /// `PerIteration` indicates that the input to the benchmark routine or the value returned from
+ /// that routine is extremely large or holds some limited resource, such that holding many values
+ /// in memory at once is infeasible. This provides the worst measurement overhead, but the
+ /// lowest memory usage.
+ ///
+ /// In testing, the maximum measurement overhead from benchmarking with `PerIteration` is on the
+ /// order of 350 nanoseconds or 350,000 picoseconds. This is presented as a rough guide; your
+ /// results may vary.
+ PerIteration,
+
+ /// `NumBatches` will attempt to divide the iterations up into a given number of batches.
+ /// A larger number of batches (and thus smaller batches) will reduce memory usage but increase
+ /// measurement overhead. This allows the user to choose their own tradeoff between memory usage
+ /// and measurement overhead, but care must be taken in tuning the number of batches. Most
+ /// benchmarks should use `SmallInput` or `LargeInput` instead.
+ NumBatches(u64),
+
+ /// `NumIterations` fixes the batch size to a constant number, specified by the user. This
+ /// allows the user to choose their own tradeoff between overhead and memory usage, but care must
+ /// be taken in tuning the batch size. In general, the measurement overhead of NumIterations
+ /// will be larger than that of `NumBatches`. Most benchmarks should use `SmallInput` or
+ /// `LargeInput` instead.
+ NumIterations(u64),
+
+ #[doc(hidden)]
+ __NonExhaustive,
+}
+impl BatchSize {
+ /// Convert to a number of iterations per batch.
+ ///
+ /// We try to do a constant number of batches regardless of the number of iterations in this
+ /// sample. If the measurement overhead is roughly constant regardless of the number of
+ /// iterations the analysis of the results later will have an easier time separating the
+ /// measurement overhead from the benchmark time.
+ fn iters_per_batch(self, iters: u64) -> u64 {
+ match self {
+ BatchSize::SmallInput => (iters + 10 - 1) / 10,
+ BatchSize::LargeInput => (iters + 1000 - 1) / 1000,
+ BatchSize::PerIteration => 1,
+ BatchSize::NumBatches(batches) => (iters + batches - 1) / batches,
+ BatchSize::NumIterations(size) => size,
+ BatchSize::__NonExhaustive => panic!("__NonExhaustive is not a valid BatchSize."),
+ }
+ }
+}
+
+/// Timer struct used to iterate a benchmarked function and measure the runtime.
+///
+/// This struct provides different timing loops as methods. Each timing loop provides a different
+/// way to time a routine and each has advantages and disadvantages.
+///
+/// * If you want to do the iteration and measurement yourself (eg. passing the iteration count
+/// to a separate process), use `iter_custom`.
+/// * If your routine requires no per-iteration setup and returns a value with an expensive `drop`
+/// method, use `iter_with_large_drop`.
+/// * If your routine requires some per-iteration setup that shouldn't be timed, use `iter_batched`
+/// or `iter_batched_ref`. See [`BatchSize`](enum.BatchSize.html) for a discussion of batch sizes.
+/// If the setup value implements `Drop` and you don't want to include the `drop` time in the
+/// measurement, use `iter_batched_ref`, otherwise use `iter_batched`. These methods are also
+/// suitable for benchmarking routines which return a value with an expensive `drop` method,
+/// but are more complex than `iter_with_large_drop`.
+/// * Otherwise, use `iter`.
+pub struct Bencher<'a, M: Measurement = WallTime> {
+ iterated: bool, // have we iterated this benchmark?
+ iters: u64, // Number of times to iterate this benchmark
+ value: M::Value, // The measured value
+ measurement: &'a M, // Reference to the measurement object
+ elapsed_time: Duration, // How much time did it take to perform the iteration? Used for the warmup period.
+}
+impl<'a, M: Measurement> Bencher<'a, M> {
+ /// Times a `routine` by executing it many times and timing the total elapsed time.
+ ///
+ /// Prefer this timing loop when `routine` returns a value that doesn't have a destructor.
+ ///
+ /// # Timing model
+ ///
+ /// Note that the `Bencher` also times the time required to destroy the output of `routine()`.
+ /// Therefore prefer this timing loop when the runtime of `mem::drop(O)` is negligible compared
+ /// to the runtime of the `routine`.
+ ///
+ /// ```text
+ /// elapsed = Instant::now + iters * (routine + mem::drop(O) + Range::next)
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// // The function to benchmark
+ /// fn foo() {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("iter", move |b| {
+ /// b.iter(|| foo())
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter<O, R>(&mut self, mut routine: R)
+ where
+ R: FnMut() -> O,
+ {
+ self.iterated = true;
+ let time_start = Instant::now();
+ let start = self.measurement.start();
+ for _ in 0..self.iters {
+ black_box(routine());
+ }
+ self.value = self.measurement.end(start);
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ /// Times a `routine` by executing it many times and relying on `routine` to measure its own execution time.
+ ///
+ /// Prefer this timing loop in cases where `routine` has to do its own measurements to
+ /// get accurate timing information (for example in multi-threaded scenarios where you spawn
+ /// and coordinate with multiple threads).
+ ///
+ /// # Timing model
+ /// Custom, the timing model is whatever is returned as the Duration from `routine`.
+ ///
+ /// # Example
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ /// use criterion::*;
+ /// use criterion::black_box;
+ /// use std::time::Instant;
+ ///
+ /// fn foo() {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("iter", move |b| {
+ /// b.iter_custom(|iters| {
+ /// let start = Instant::now();
+ /// for _i in 0..iters {
+ /// black_box(foo());
+ /// }
+ /// start.elapsed()
+ /// })
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_custom<R>(&mut self, mut routine: R)
+ where
+ R: FnMut(u64) -> M::Value,
+ {
+ self.iterated = true;
+ let time_start = Instant::now();
+ self.value = routine(self.iters);
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ #[doc(hidden)]
+ pub fn iter_with_setup<I, O, S, R>(&mut self, setup: S, routine: R)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> O,
+ {
+ self.iter_batched(setup, routine, BatchSize::PerIteration);
+ }
+
+ /// Times a `routine` by collecting its output on each iteration. This avoids timing the
+ /// destructor of the value returned by `routine`.
+ ///
+ /// WARNING: This requires `O(iters * mem::size_of::<O>())` of memory, and `iters` is not under the
+ /// control of the caller. If this causes out-of-memory errors, use `iter_batched` instead.
+ ///
+ /// # Timing model
+ ///
+ /// ``` text
+ /// elapsed = Instant::now + iters * (routine) + Iterator::collect::<Vec<_>>
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// fn create_vector() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("with_drop", move |b| {
+ /// // This will avoid timing the Vec::drop.
+ /// b.iter_with_large_drop(|| create_vector())
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ pub fn iter_with_large_drop<O, R>(&mut self, mut routine: R)
+ where
+ R: FnMut() -> O,
+ {
+ self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput);
+ }
+
+ #[doc(hidden)]
+ pub fn iter_with_large_setup<I, O, S, R>(&mut self, setup: S, routine: R)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> O,
+ {
+ self.iter_batched(setup, routine, BatchSize::NumBatches(1));
+ }
+
+ /// Times a `routine` that requires some input by generating a batch of input, then timing the
+ /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
+ /// details on choosing the batch size. Use this when the routine must consume its input.
+ ///
+ /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
+ /// data on each iteration.
+ ///
+ /// # Timing model
+ ///
+ /// ```text
+ /// elapsed = (Instant::now * num_batches) + (iters * (routine + O::drop)) + Vec::extend
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// fn create_scrambled_data() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// // The sorting algorithm to test
+ /// fn sort(data: &mut [u64]) {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let data = create_scrambled_data();
+ ///
+ /// c.bench_function("with_setup", move |b| {
+ /// // This will avoid timing the to_vec call.
+ /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput)
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_batched<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> O,
+ {
+ self.iterated = true;
+ let batch_size = size.iters_per_batch(self.iters);
+ assert!(batch_size != 0, "Batch size must not be zero.");
+ let time_start = Instant::now();
+ self.value = self.measurement.zero();
+
+ if batch_size == 1 {
+ for _ in 0..self.iters {
+ let input = black_box(setup());
+
+ let start = self.measurement.start();
+ let output = routine(input);
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ drop(black_box(output));
+ }
+ } else {
+ let mut iteration_counter = 0;
+
+ while iteration_counter < self.iters {
+ let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter);
+
+ let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
+ let mut outputs = Vec::with_capacity(batch_size as usize);
+
+ let start = self.measurement.start();
+ outputs.extend(inputs.into_iter().map(&mut routine));
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ black_box(outputs);
+
+ iteration_counter += batch_size;
+ }
+ }
+
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ /// Times a `routine` that requires some input by generating a batch of input, then timing the
+ /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
+ /// details on choosing the batch size. Use this when the routine should accept the input by
+ /// mutable reference.
+ ///
+ /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
+ /// data on each iteration.
+ ///
+ /// # Timing model
+ ///
+ /// ```text
+ /// elapsed = (Instant::now * num_batches) + (iters * routine) + Vec::extend
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// fn create_scrambled_data() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// // The sorting algorithm to test
+ /// fn sort(data: &mut [u64]) {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let data = create_scrambled_data();
+ ///
+ /// c.bench_function("with_setup", move |b| {
+ /// // This will avoid timing the to_vec call.
+ /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput)
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_batched_ref<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
+ where
+ S: FnMut() -> I,
+ R: FnMut(&mut I) -> O,
+ {
+ self.iterated = true;
+ let batch_size = size.iters_per_batch(self.iters);
+ assert!(batch_size != 0, "Batch size must not be zero.");
+ let time_start = Instant::now();
+ self.value = self.measurement.zero();
+
+ if batch_size == 1 {
+ for _ in 0..self.iters {
+ let mut input = black_box(setup());
+
+ let start = self.measurement.start();
+ let output = routine(&mut input);
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ drop(black_box(output));
+ drop(black_box(input));
+ }
+ } else {
+ let mut iteration_counter = 0;
+
+ while iteration_counter < self.iters {
+ let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter);
+
+ let mut inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
+ let mut outputs = Vec::with_capacity(batch_size as usize);
+
+ let start = self.measurement.start();
+ outputs.extend(inputs.iter_mut().map(&mut routine));
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ black_box(outputs);
+
+ iteration_counter += batch_size;
+ }
+ }
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ // Benchmarks must actually call one of the iter methods. This causes benchmarks to fail loudly
+ // if they don't.
+ fn assert_iterated(&mut self) {
+ if !self.iterated {
+ panic!("Benchmark function must call Bencher::iter or related method.");
+ }
+ self.iterated = false;
+ }
+}
+
+/// Baseline describes how the baseline_directory is handled.
+#[derive(Debug, Clone, Copy)]
+pub enum Baseline {
+ /// Compare ensures a previous saved version of the baseline
+ /// exists and runs comparison against that.
+ Compare,
+ /// Save writes the benchmark results to the baseline directory,
+ /// overwriting any results that were previously there.
+ Save,
+}
+
+/// Enum used to select the plotting backend.
+#[derive(Debug, Clone, Copy)]
+pub enum PlottingBackend {
+ /// Plotting backend which uses the external `gnuplot` command to render plots. This is the
+ /// default if the `gnuplot` command is installed.
+ Gnuplot,
+ /// Plotting backend which uses the rust 'Plotters' library. This is the default if `gnuplot`
+ /// is not installed.
+ Plotters,
+}
+
+#[derive(Debug, Clone)]
+/// Enum representing the execution mode.
+pub(crate) enum Mode {
+ /// Run benchmarks normally
+ Benchmark,
+ /// List all benchmarks but do not run them.
+ List,
+ /// Run bennchmarks once to verify that they work, but otherwise do not measure them.
+ Test,
+ /// Iterate benchmarks for a given length of time but do not analyze or report on them.
+ Profile(Duration),
+}
+impl Mode {
+ pub fn is_benchmark(&self) -> bool {
+ match self {
+ Mode::Benchmark => true,
+ _ => false,
+ }
+ }
+}
+
+/// The benchmark manager
+///
+/// `Criterion` lets you configure and execute benchmarks
+///
+/// Each benchmark consists of four phases:
+///
+/// - **Warm-up**: The routine is repeatedly executed, to let the CPU/OS/JIT/interpreter adapt to
+/// the new load
+/// - **Measurement**: The routine is repeatedly executed, and timing information is collected into
+/// a sample
+/// - **Analysis**: The sample is analyzed and distiled into meaningful statistics that get
+/// reported to stdout, stored in files, and plotted
+/// - **Comparison**: The current sample is compared with the sample obtained in the previous
+/// benchmark.
+pub struct Criterion<M: Measurement = WallTime> {
+ config: BenchmarkConfig,
+ plotting_backend: PlottingBackend,
+ plotting_enabled: bool,
+ filter: Option<Regex>,
+ report: Box<dyn Report>,
+ output_directory: PathBuf,
+ baseline_directory: String,
+ baseline: Baseline,
+ load_baseline: Option<String>,
+ all_directories: HashSet<String>,
+ all_titles: HashSet<String>,
+ measurement: M,
+ profiler: Box<RefCell<dyn Profiler>>,
+ connection: Option<MutexGuard<'static, Connection>>,
+ mode: Mode,
+}
+
+impl Default for Criterion {
+ /// Creates a benchmark manager with the following default settings:
+ ///
+ /// - Sample size: 100 measurements
+ /// - Warm-up time: 3 s
+ /// - Measurement time: 5 s
+ /// - Bootstrap size: 100 000 resamples
+ /// - Noise threshold: 0.01 (1%)
+ /// - Confidence level: 0.95
+ /// - Significance level: 0.05
+ /// - Plotting: enabled, using gnuplot if available or plotters if gnuplot is not available
+ /// - No filter
+ fn default() -> Criterion {
+ let mut reports: Vec<Box<dyn Report>> = vec![];
+ if CARGO_CRITERION_CONNECTION.is_none() {
+ reports.push(Box::new(CliReport::new(false, false, false)));
+ }
+ reports.push(Box::new(FileCsvReport));
+
+ // Set criterion home to (in descending order of preference):
+ // - $CRITERION_HOME (cargo-criterion sets this, but other users could as well)
+ // - $CARGO_TARGET_DIR/criterion
+ // - ./target/criterion
+ let output_directory = if let Some(value) = std::env::var_os("CRITERION_HOME") {
+ PathBuf::from(value)
+ } else if let Some(value) = std::env::var_os("CARGO_TARGET_DIR") {
+ PathBuf::from(value).join("criterion")
+ } else {
+ PathBuf::from("target/criterion")
+ };
+
+ Criterion {
+ config: BenchmarkConfig {
+ confidence_level: 0.95,
+ measurement_time: Duration::new(5, 0),
+ noise_threshold: 0.01,
+ nresamples: 100_000,
+ sample_size: 100,
+ significance_level: 0.05,
+ warm_up_time: Duration::new(3, 0),
+ sampling_mode: SamplingMode::Auto,
+ },
+ plotting_backend: *DEFAULT_PLOTTING_BACKEND,
+ plotting_enabled: true,
+ filter: None,
+ report: Box::new(Reports::new(reports)),
+ baseline_directory: "base".to_owned(),
+ baseline: Baseline::Save,
+ load_baseline: None,
+ output_directory,
+ all_directories: HashSet::new(),
+ all_titles: HashSet::new(),
+ measurement: WallTime,
+ profiler: Box::new(RefCell::new(ExternalProfiler)),
+ connection: CARGO_CRITERION_CONNECTION
+ .as_ref()
+ .map(|mtx| mtx.lock().unwrap()),
+ mode: Mode::Benchmark,
+ }
+ }
+}
+
+impl<M: Measurement> Criterion<M> {
+ /// Changes the measurement for the benchmarks run with this runner. See the
+ /// Measurement trait for more details
+ pub fn with_measurement<M2: Measurement>(self, m: M2) -> Criterion<M2> {
+ // Can't use struct update syntax here because they're technically different types.
+ Criterion {
+ config: self.config,
+ plotting_backend: self.plotting_backend,
+ plotting_enabled: self.plotting_enabled,
+ filter: self.filter,
+ report: self.report,
+ baseline_directory: self.baseline_directory,
+ baseline: self.baseline,
+ load_baseline: self.load_baseline,
+ output_directory: self.output_directory,
+ all_directories: self.all_directories,
+ all_titles: self.all_titles,
+ measurement: m,
+ profiler: self.profiler,
+ connection: self.connection,
+ mode: self.mode,
+ }
+ }
+
+ /// Changes the internal profiler for benchmarks run with this runner. See
+ /// the Profiler trait for more details.
+ pub fn with_profiler<P: Profiler + 'static>(self, p: P) -> Criterion<M> {
+ Criterion {
+ profiler: Box::new(RefCell::new(p)),
+ ..self
+ }
+ }
+
+ /// Set the plotting backend. By default, Criterion will use gnuplot if available, or plotters
+ /// if not.
+ ///
+ /// Panics if `backend` is `PlottingBackend::Gnuplot` and gnuplot is not available.
+ pub fn plotting_backend(self, backend: PlottingBackend) -> Criterion<M> {
+ if let PlottingBackend::Gnuplot = backend {
+ if GNUPLOT_VERSION.is_err() {
+ panic!("Gnuplot plotting backend was requested, but gnuplot is not available. To continue, either install Gnuplot or allow Criterion.rs to fall back to using plotters.");
+ }
+ }
+
+ Criterion {
+ plotting_backend: backend,
+ ..self
+ }
+ }
+
+ /// Changes the default size of the sample for benchmarks run with this runner.
+ ///
+ /// A bigger sample should yield more accurate results if paired with a sufficiently large
+ /// measurement time.
+ ///
+ /// Sample size must be at least 10.
+ ///
+ /// # Panics
+ ///
+ /// Panics if n < 10
+ pub fn sample_size(mut self, n: usize) -> Criterion<M> {
+ assert!(n >= 10);
+
+ self.config.sample_size = n;
+ self
+ }
+
+ /// Changes the default warm up time for benchmarks run with this runner.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the input duration is zero
+ pub fn warm_up_time(mut self, dur: Duration) -> Criterion<M> {
+ assert!(dur.to_nanos() > 0);
+
+ self.config.warm_up_time = dur;
+ self
+ }
+
+ /// Changes the default measurement time for benchmarks run with this runner.
+ ///
+ /// With a longer time, the measurement will become more resilient to transitory peak loads
+ /// caused by external programs
+ ///
+ /// **Note**: If the measurement time is too "low", Criterion will automatically increase it
+ ///
+ /// # Panics
+ ///
+ /// Panics if the input duration in zero
+ pub fn measurement_time(mut self, dur: Duration) -> Criterion<M> {
+ assert!(dur.to_nanos() > 0);
+
+ self.config.measurement_time = dur;
+ self
+ }
+
+ /// Changes the default number of resamples for benchmarks run with this runner.
+ ///
+ /// Number of resamples to use for the
+ /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling)
+ ///
+ /// A larger number of resamples reduces the random sampling errors, which are inherent to the
+ /// bootstrap method, but also increases the analysis time
+ ///
+ /// # Panics
+ ///
+ /// Panics if the number of resamples is set to zero
+ pub fn nresamples(mut self, n: usize) -> Criterion<M> {
+ assert!(n > 0);
+ if n <= 1000 {
+ println!("\nWarning: It is not recommended to reduce nresamples below 1000.");
+ }
+
+ self.config.nresamples = n;
+ self
+ }
+
+ /// Changes the default noise threshold for benchmarks run with this runner. The noise threshold
+ /// is used to filter out small changes in performance, even if they are statistically
+ /// significant. Sometimes benchmarking the same code twice will result in small but
+ /// statistically significant differences solely because of noise. This provides a way to filter
+ /// out some of these false positives at the cost of making it harder to detect small changes
+ /// to the true performance of the benchmark.
+ ///
+ /// The default is 0.01, meaning that changes smaller than 1% will be ignored.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the threshold is set to a negative value
+ pub fn noise_threshold(mut self, threshold: f64) -> Criterion<M> {
+ assert!(threshold >= 0.0);
+
+ self.config.noise_threshold = threshold;
+ self
+ }
+
+ /// Changes the default confidence level for benchmarks run with this runner. The confidence
+ /// level is the desired probability that the true runtime lies within the estimated
+ /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is
+ /// 0.95, meaning that the confidence interval should capture the true value 95% of the time.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the confidence level is set to a value outside the `(0, 1)` range
+ pub fn confidence_level(mut self, cl: f64) -> Criterion<M> {
+ assert!(cl > 0.0 && cl < 1.0);
+ if cl < 0.5 {
+ println!("\nWarning: It is not recommended to reduce confidence level below 0.5.");
+ }
+
+ self.config.confidence_level = cl;
+ self
+ }
+
+ /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance)
+ /// for benchmarks run with this runner. This is used to perform a
+ /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if
+ /// the measurements from this run are different from the measured performance of the last run.
+ /// The significance level is the desired probability that two measurements of identical code
+ /// will be considered 'different' due to noise in the measurements. The default value is 0.05,
+ /// meaning that approximately 5% of identical benchmarks will register as different due to
+ /// noise.
+ ///
+ /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
+ /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to
+ /// detect small but real changes in the performance. By setting the significance level
+ /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
+ /// report more spurious differences.
+ ///
+ /// See also the noise threshold setting.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the significance level is set to a value outside the `(0, 1)` range
+ pub fn significance_level(mut self, sl: f64) -> Criterion<M> {
+ assert!(sl > 0.0 && sl < 1.0);
+
+ self.config.significance_level = sl;
+ self
+ }
+
+ fn create_plotter(&self) -> Box<dyn Plotter> {
+ match self.plotting_backend {
+ PlottingBackend::Gnuplot => Box::new(Gnuplot::default()),
+ PlottingBackend::Plotters => Box::new(PlottersBackend::default()),
+ }
+ }
+
+ /// Enables plotting
+ pub fn with_plots(mut self) -> Criterion<M> {
+ self.plotting_enabled = true;
+ let mut reports: Vec<Box<dyn Report>> = vec![];
+ if self.connection.is_none() {
+ reports.push(Box::new(CliReport::new(false, false, false)));
+ }
+ reports.push(Box::new(FileCsvReport));
+ reports.push(Box::new(Html::new(self.create_plotter())));
+ self.report = Box::new(Reports::new(reports));
+
+ self
+ }
+
+ /// Disables plotting
+ pub fn without_plots(mut self) -> Criterion<M> {
+ self.plotting_enabled = false;
+ let mut reports: Vec<Box<dyn Report>> = vec![];
+ if self.connection.is_none() {
+ reports.push(Box::new(CliReport::new(false, false, false)));
+ }
+ reports.push(Box::new(FileCsvReport));
+ self.report = Box::new(Reports::new(reports));
+ self
+ }
+
+ /// Return true if generation of the plots is possible.
+ pub fn can_plot(&self) -> bool {
+ // Trivially true now that we have plotters.
+ // TODO: Deprecate and remove this.
+ true
+ }
+
+ /// Names an explicit baseline and enables overwriting the previous results.
+ pub fn save_baseline(mut self, baseline: String) -> Criterion<M> {
+ self.baseline_directory = baseline;
+ self.baseline = Baseline::Save;
+ self
+ }
+
+ /// Names an explicit baseline and disables overwriting the previous results.
+ pub fn retain_baseline(mut self, baseline: String) -> Criterion<M> {
+ self.baseline_directory = baseline;
+ self.baseline = Baseline::Compare;
+ self
+ }
+
+ /// Filters the benchmarks. Only benchmarks with names that contain the
+ /// given string will be executed.
+ pub fn with_filter<S: Into<String>>(mut self, filter: S) -> Criterion<M> {
+ let filter_text = filter.into();
+ let filter = Regex::new(&filter_text).unwrap_or_else(|err| {
+ panic!(
+ "Unable to parse '{}' as a regular expression: {}",
+ filter_text, err
+ )
+ });
+ self.filter = Some(filter);
+
+ self
+ }
+
+ /// Set the output directory (currently for testing only)
+ #[doc(hidden)]
+ pub fn output_directory(mut self, path: &Path) -> Criterion<M> {
+ self.output_directory = path.to_owned();
+
+ self
+ }
+
+ /// Set the profile time (currently for testing only)
+ #[doc(hidden)]
+ pub fn profile_time(mut self, profile_time: Option<Duration>) -> Criterion<M> {
+ match profile_time {
+ Some(time) => self.mode = Mode::Profile(time),
+ None => self.mode = Mode::Benchmark,
+ }
+
+ self
+ }
+
+ /// Generate the final summary at the end of a run.
+ #[doc(hidden)]
+ pub fn final_summary(&self) {
+ if !self.mode.is_benchmark() {
+ return;
+ }
+
+ let report_context = ReportContext {
+ output_directory: self.output_directory.clone(),
+ plot_config: PlotConfiguration::default(),
+ };
+
+ self.report.final_summary(&report_context);
+ }
+
+ /// Configure this criterion struct based on the command-line arguments to
+ /// this process.
+ #[cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))]
+ pub fn configure_from_args(mut self) -> Criterion<M> {
+ use clap::{App, Arg};
+ let matches = App::new("Criterion Benchmark")
+ .arg(Arg::with_name("FILTER")
+ .help("Skip benchmarks whose names do not contain FILTER.")
+ .index(1))
+ .arg(Arg::with_name("color")
+ .short("c")
+ .long("color")
+ .alias("colour")
+ .takes_value(true)
+ .possible_values(&["auto", "always", "never"])
+ .default_value("auto")
+ .help("Configure coloring of output. always = always colorize output, never = never colorize output, auto = colorize output if output is a tty and compiled for unix."))
+ .arg(Arg::with_name("verbose")
+ .short("v")
+ .long("verbose")
+ .help("Print additional statistical information."))
+ .arg(Arg::with_name("noplot")
+ .short("n")
+ .long("noplot")
+ .help("Disable plot and HTML generation."))
+ .arg(Arg::with_name("save-baseline")
+ .short("s")
+ .long("save-baseline")
+ .default_value("base")
+ .help("Save results under a named baseline."))
+ .arg(Arg::with_name("baseline")
+ .short("b")
+ .long("baseline")
+ .takes_value(true)
+ .conflicts_with("save-baseline")
+ .help("Compare to a named baseline."))
+ .arg(Arg::with_name("list")
+ .long("list")
+ .help("List all benchmarks")
+ .conflicts_with_all(&["test", "profile-time"]))
+ .arg(Arg::with_name("profile-time")
+ .long("profile-time")
+ .takes_value(true)
+ .help("Iterate each benchmark for approximately the given number of seconds, doing no analysis and without storing the results. Useful for running the benchmarks in a profiler.")
+ .conflicts_with_all(&["test", "list"]))
+ .arg(Arg::with_name("load-baseline")
+ .long("load-baseline")
+ .takes_value(true)
+ .conflicts_with("profile-time")
+ .requires("baseline")
+ .help("Load a previous baseline instead of sampling new data."))
+ .arg(Arg::with_name("sample-size")
+ .long("sample-size")
+ .takes_value(true)
+ .help(&format!("Changes the default size of the sample for this run. [default: {}]", self.config.sample_size)))
+ .arg(Arg::with_name("warm-up-time")
+ .long("warm-up-time")
+ .takes_value(true)
+ .help(&format!("Changes the default warm up time for this run. [default: {}]", self.config.warm_up_time.as_secs())))
+ .arg(Arg::with_name("measurement-time")
+ .long("measurement-time")
+ .takes_value(true)
+ .help(&format!("Changes the default measurement time for this run. [default: {}]", self.config.measurement_time.as_secs())))
+ .arg(Arg::with_name("nresamples")
+ .long("nresamples")
+ .takes_value(true)
+ .help(&format!("Changes the default number of resamples for this run. [default: {}]", self.config.nresamples)))
+ .arg(Arg::with_name("noise-threshold")
+ .long("noise-threshold")
+ .takes_value(true)
+ .help(&format!("Changes the default noise threshold for this run. [default: {}]", self.config.noise_threshold)))
+ .arg(Arg::with_name("confidence-level")
+ .long("confidence-level")
+ .takes_value(true)
+ .help(&format!("Changes the default confidence level for this run. [default: {}]", self.config.confidence_level)))
+ .arg(Arg::with_name("significance-level")
+ .long("significance-level")
+ .takes_value(true)
+ .help(&format!("Changes the default significance level for this run. [default: {}]", self.config.significance_level)))
+ .arg(Arg::with_name("test")
+ .hidden(true)
+ .long("test")
+ .help("Run the benchmarks once, to verify that they execute successfully, but do not measure or report the results.")
+ .conflicts_with_all(&["list", "profile-time"]))
+ .arg(Arg::with_name("bench")
+ .hidden(true)
+ .long("bench"))
+ .arg(Arg::with_name("plotting-backend")
+ .long("plotting-backend")
+ .takes_value(true)
+ .possible_values(&["gnuplot", "plotters"])
+ .help("Set the plotting backend. By default, Criterion.rs will use the gnuplot backend if gnuplot is available, or the plotters backend if it isn't."))
+ .arg(Arg::with_name("output-format")
+ .long("output-format")
+ .takes_value(true)
+ .possible_values(&["criterion", "bencher"])
+ .default_value("criterion")
+ .help("Change the CLI output format. By default, Criterion.rs will use its own format. If output format is set to 'bencher', Criterion.rs will print output in a format that resembles the 'bencher' crate."))
+ .arg(Arg::with_name("nocapture")
+ .long("nocapture")
+ .hidden(true)
+ .help("Ignored, but added for compatibility with libtest."))
+ .arg(Arg::with_name("version")
+ .hidden(true)
+ .short("V")
+ .long("version"))
+ .after_help("
+This executable is a Criterion.rs benchmark.
+See https://github.com/bheisler/criterion.rs for more details.
+
+To enable debug output, define the environment variable CRITERION_DEBUG.
+Criterion.rs will output more debug information and will save the gnuplot
+scripts alongside the generated plots.
+
+To test that the benchmarks work, run `cargo test --benches`
+")
+ .get_matches();
+
+ if self.connection.is_some() {
+ if let Some(color) = matches.value_of("color") {
+ if color != "auto" {
+ println!("Warning: --color will be ignored when running with cargo-criterion. Use `cargo criterion --color {} -- <args>` instead.", color);
+ }
+ }
+ if matches.is_present("verbose") {
+ println!("Warning: --verbose will be ignored when running with cargo-criterion. Use `cargo criterion --output-format verbose -- <args>` instead.");
+ }
+ if matches.is_present("noplot") {
+ println!("Warning: --noplot will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend disabled -- <args>` instead.");
+ }
+ if let Some(backend) = matches.value_of("plotting-backend") {
+ println!("Warning: --plotting-backend will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend {} -- <args>` instead.", backend);
+ }
+ if let Some(format) = matches.value_of("output-format") {
+ if format != "criterion" {
+ println!("Warning: --output-format will be ignored when running with cargo-criterion. Use `cargo criterion --output-format {} -- <args>` instead.", format);
+ }
+ }
+
+ if matches.is_present("baseline")
+ || matches
+ .value_of("save-baseline")
+ .map(|base| base != "base")
+ .unwrap_or(false)
+ || matches.is_present("load-baseline")
+ {
+ println!("Error: baselines are not supported when running with cargo-criterion.");
+ std::process::exit(1);
+ }
+ }
+
+ let bench = matches.is_present("bench");
+ let test = matches.is_present("test");
+ let test_mode = match (bench, test) {
+ (true, true) => true, // cargo bench -- --test should run tests
+ (true, false) => false, // cargo bench should run benchmarks
+ (false, _) => true, // cargo test --benches should run tests
+ };
+
+ self.mode = if test_mode {
+ Mode::Test
+ } else if matches.is_present("list") {
+ Mode::List
+ } else if matches.is_present("profile-time") {
+ let num_seconds = value_t!(matches.value_of("profile-time"), u64).unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ if num_seconds < 1 {
+ println!("Profile time must be at least one second.");
+ std::process::exit(1);
+ }
+
+ Mode::Profile(Duration::from_secs(num_seconds))
+ } else {
+ Mode::Benchmark
+ };
+
+ // This is kind of a hack, but disable the connection to the runner if we're not benchmarking.
+ if !self.mode.is_benchmark() {
+ self.connection = None;
+ }
+
+ if let Some(filter) = matches.value_of("FILTER") {
+ self = self.with_filter(filter);
+ }
+
+ match matches.value_of("plotting-backend") {
+ // Use plotting_backend() here to re-use the panic behavior if Gnuplot is not available.
+ Some("gnuplot") => self = self.plotting_backend(PlottingBackend::Gnuplot),
+ Some("plotters") => self = self.plotting_backend(PlottingBackend::Plotters),
+ Some(val) => panic!("Unexpected plotting backend '{}'", val),
+ None => {}
+ }
+
+ if matches.is_present("noplot") {
+ self = self.without_plots();
+ } else {
+ self = self.with_plots();
+ }
+
+ if let Some(dir) = matches.value_of("save-baseline") {
+ self.baseline = Baseline::Save;
+ self.baseline_directory = dir.to_owned()
+ }
+ if let Some(dir) = matches.value_of("baseline") {
+ self.baseline = Baseline::Compare;
+ self.baseline_directory = dir.to_owned();
+ }
+
+ if self.connection.is_some() {
+ // disable all reports when connected to cargo-criterion; it will do the reporting.
+ self.report = Box::new(Reports::new(vec![]));
+ } else {
+ let cli_report: Box<dyn Report> = match matches.value_of("output-format") {
+ Some("bencher") => Box::new(BencherReport),
+ _ => {
+ let verbose = matches.is_present("verbose");
+ let stdout_isatty = atty::is(atty::Stream::Stdout);
+ let mut enable_text_overwrite = stdout_isatty && !verbose && !debug_enabled();
+ let enable_text_coloring;
+ match matches.value_of("color") {
+ Some("always") => {
+ enable_text_coloring = true;
+ }
+ Some("never") => {
+ enable_text_coloring = false;
+ enable_text_overwrite = false;
+ }
+ _ => enable_text_coloring = stdout_isatty,
+ };
+ Box::new(CliReport::new(
+ enable_text_overwrite,
+ enable_text_coloring,
+ verbose,
+ ))
+ }
+ };
+
+ let mut reports: Vec<Box<dyn Report>> = vec![];
+ reports.push(cli_report);
+ reports.push(Box::new(FileCsvReport));
+ if self.plotting_enabled {
+ reports.push(Box::new(Html::new(self.create_plotter())));
+ }
+ self.report = Box::new(Reports::new(reports));
+ }
+
+ if let Some(dir) = matches.value_of("load-baseline") {
+ self.load_baseline = Some(dir.to_owned());
+ }
+
+ if matches.is_present("sample-size") {
+ let num_size = value_t!(matches.value_of("sample-size"), usize).unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ assert!(num_size >= 10);
+ self.config.sample_size = num_size;
+ }
+ if matches.is_present("warm-up-time") {
+ let num_seconds = value_t!(matches.value_of("warm-up-time"), u64).unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ let dur = std::time::Duration::new(num_seconds, 0);
+ assert!(dur.to_nanos() > 0);
+
+ self.config.warm_up_time = dur;
+ }
+ if matches.is_present("measurement-time") {
+ let num_seconds =
+ value_t!(matches.value_of("measurement-time"), u64).unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ let dur = std::time::Duration::new(num_seconds, 0);
+ assert!(dur.to_nanos() > 0);
+
+ self.config.measurement_time = dur;
+ }
+ if matches.is_present("nresamples") {
+ let num_resamples =
+ value_t!(matches.value_of("nresamples"), usize).unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ assert!(num_resamples > 0);
+
+ self.config.nresamples = num_resamples;
+ }
+ if matches.is_present("noise-threshold") {
+ let num_noise_threshold = value_t!(matches.value_of("noise-threshold"), f64)
+ .unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ assert!(num_noise_threshold > 0.0);
+
+ self.config.noise_threshold = num_noise_threshold;
+ }
+ if matches.is_present("confidence-level") {
+ let num_confidence_level = value_t!(matches.value_of("confidence-level"), f64)
+ .unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ assert!(num_confidence_level > 0.0 && num_confidence_level < 1.0);
+
+ self.config.confidence_level = num_confidence_level;
+ }
+ if matches.is_present("significance-level") {
+ let num_significance_level = value_t!(matches.value_of("significance-level"), f64)
+ .unwrap_or_else(|e| {
+ println!("{}", e);
+ std::process::exit(1)
+ });
+
+ assert!(num_significance_level > 0.0 && num_significance_level < 1.0);
+
+ self.config.significance_level = num_significance_level;
+ }
+
+ self
+ }
+
+ fn filter_matches(&self, id: &str) -> bool {
+ match self.filter {
+ Some(ref regex) => regex.is_match(id),
+ None => true,
+ }
+ }
+
+ /// Return a benchmark group. All benchmarks performed using a benchmark group will be
+ /// grouped together in the final report.
+ ///
+ /// # Examples:
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ /// use self::criterion::*;
+ ///
+ /// fn bench_simple(c: &mut Criterion) {
+ /// let mut group = c.benchmark_group("My Group");
+ ///
+ /// // Now we can perform benchmarks with this group
+ /// group.bench_function("Bench 1", |b| b.iter(|| 1 ));
+ /// group.bench_function("Bench 2", |b| b.iter(|| 2 ));
+ ///
+ /// group.finish();
+ /// }
+ /// criterion_group!(benches, bench_simple);
+ /// criterion_main!(benches);
+ /// ```
+ /// # Panics:
+ /// Panics if the group name is empty
+ pub fn benchmark_group<S: Into<String>>(&mut self, group_name: S) -> BenchmarkGroup<'_, M> {
+ let group_name = group_name.into();
+ if group_name.is_empty() {
+ panic!("Group name must not be empty.");
+ }
+
+ if let Some(conn) = &self.connection {
+ conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: &group_name })
+ .unwrap();
+ }
+
+ BenchmarkGroup::new(self, group_name)
+ }
+}
+impl<M> Criterion<M>
+where
+ M: Measurement + 'static,
+{
+ /// Benchmarks a function. For comparing multiple functions, see `benchmark_group`.
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ /// use self::criterion::*;
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// // Setup (construct data, allocate memory, etc)
+ /// c.bench_function(
+ /// "function_name",
+ /// |b| b.iter(|| {
+ /// // Code to benchmark goes here
+ /// }),
+ /// );
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ pub fn bench_function<F>(&mut self, id: &str, f: F) -> &mut Criterion<M>
+ where
+ F: FnMut(&mut Bencher<'_, M>),
+ {
+ self.benchmark_group(id)
+ .bench_function(BenchmarkId::no_function(), f);
+ self
+ }
+
+ /// Benchmarks a function with an input. For comparing multiple functions or multiple inputs,
+ /// see `benchmark_group`.
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ /// use self::criterion::*;
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// // Setup (construct data, allocate memory, etc)
+ /// let input = 5u64;
+ /// c.bench_with_input(
+ /// BenchmarkId::new("function_name", input), &input,
+ /// |b, i| b.iter(|| {
+ /// // Code to benchmark using input `i` goes here
+ /// }),
+ /// );
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ pub fn bench_with_input<F, I>(&mut self, id: BenchmarkId, input: &I, f: F) -> &mut Criterion<M>
+ where
+ F: FnMut(&mut Bencher<'_, M>, &I),
+ {
+ // Guaranteed safe because external callers can't create benchmark IDs without a function
+ // name or parameter
+ let group_name = id.function_name.unwrap();
+ let parameter = id.parameter.unwrap();
+ self.benchmark_group(group_name).bench_with_input(
+ BenchmarkId::no_function_with_input(parameter),
+ input,
+ f,
+ );
+ self
+ }
+
+ /// Benchmarks a function under various inputs
+ ///
+ /// This is a convenience method to execute several related benchmarks. Each benchmark will
+ /// receive the id: `${id}/${input}`.
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// # #[macro_use] extern crate criterion;
+ /// # use self::criterion::*;
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function_over_inputs("from_elem",
+ /// |b: &mut Bencher, size: &usize| {
+ /// b.iter(|| vec![0u8; *size]);
+ /// },
+ /// vec![1024, 2048, 4096]
+ /// );
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ #[doc(hidden)] // Soft-deprecated, use benchmark groups instead
+ pub fn bench_function_over_inputs<I, F>(
+ &mut self,
+ id: &str,
+ f: F,
+ inputs: I,
+ ) -> &mut Criterion<M>
+ where
+ I: IntoIterator,
+ I::Item: fmt::Debug + 'static,
+ F: FnMut(&mut Bencher<'_, M>, &I::Item) + 'static,
+ {
+ self.bench(id, ParameterizedBenchmark::new(id, f, inputs))
+ }
+
+ /// Benchmarks multiple functions
+ ///
+ /// All functions get the same input and are compared with the other implementations.
+ /// Works similar to `bench_function`, but with multiple functions.
+ ///
+ /// # Example
+ ///
+ /// ``` rust
+ /// # #[macro_use] extern crate criterion;
+ /// # use self::criterion::*;
+ /// # fn seq_fib(i: &u32) {}
+ /// # fn par_fib(i: &u32) {}
+ ///
+ /// fn bench_seq_fib(b: &mut Bencher, i: &u32) {
+ /// b.iter(|| {
+ /// seq_fib(i);
+ /// });
+ /// }
+ ///
+ /// fn bench_par_fib(b: &mut Bencher, i: &u32) {
+ /// b.iter(|| {
+ /// par_fib(i);
+ /// });
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let sequential_fib = Fun::new("Sequential", bench_seq_fib);
+ /// let parallel_fib = Fun::new("Parallel", bench_par_fib);
+ /// let funs = vec![sequential_fib, parallel_fib];
+ ///
+ /// c.bench_functions("Fibonacci", funs, 14);
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ #[doc(hidden)] // Soft-deprecated, use benchmark groups instead
+ pub fn bench_functions<I>(
+ &mut self,
+ id: &str,
+ funs: Vec<Fun<I, M>>,
+ input: I,
+ ) -> &mut Criterion<M>
+ where
+ I: fmt::Debug + 'static,
+ {
+ let benchmark = ParameterizedBenchmark::with_functions(
+ funs.into_iter().map(|fun| fun.f).collect(),
+ vec![input],
+ );
+
+ self.bench(id, benchmark)
+ }
+
+ /// Executes the given benchmark. Use this variant to execute benchmarks
+ /// with complex configuration. This can be used to compare multiple
+ /// functions, execute benchmarks with custom configuration settings and
+ /// more. See the Benchmark and ParameterizedBenchmark structs for more
+ /// information.
+ ///
+ /// ```rust
+ /// # #[macro_use] extern crate criterion;
+ /// # use criterion::*;
+ /// # fn routine_1() {}
+ /// # fn routine_2() {}
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// // Setup (construct data, allocate memory, etc)
+ /// c.bench(
+ /// "routines",
+ /// Benchmark::new("routine_1", |b| b.iter(|| routine_1()))
+ /// .with_function("routine_2", |b| b.iter(|| routine_2()))
+ /// .sample_size(50)
+ /// );
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ #[doc(hidden)] // Soft-deprecated, use benchmark groups instead
+ pub fn bench<B: BenchmarkDefinition<M>>(
+ &mut self,
+ group_id: &str,
+ benchmark: B,
+ ) -> &mut Criterion<M> {
+ benchmark.run(group_id, self);
+ self
+ }
+}
+
+trait DurationExt {
+ fn to_nanos(&self) -> u64;
+}
+
+const NANOS_PER_SEC: u64 = 1_000_000_000;
+
+impl DurationExt for Duration {
+ fn to_nanos(&self) -> u64 {
+ self.as_secs() * NANOS_PER_SEC + u64::from(self.subsec_nanos())
+ }
+}
+
+/// Enum representing different ways of measuring the throughput of benchmarked code.
+/// If the throughput setting is configured for a benchmark then the estimated throughput will
+/// be reported as well as the time per iteration.
+// TODO: Remove serialize/deserialize from the public API.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum Throughput {
+ /// Measure throughput in terms of bytes/second. The value should be the number of bytes
+ /// processed by one iteration of the benchmarked code. Typically, this would be the length of
+ /// an input string or `&[u8]`.
+ Bytes(u64),
+
+ /// Measure throughput in terms of elements/second. The value should be the number of elements
+ /// processed by one iteration of the benchmarked code. Typically, this would be the size of a
+ /// collection, but could also be the number of lines of input text or the number of values to
+ /// parse.
+ Elements(u64),
+}
+
+/// Axis scaling type
+#[derive(Debug, Clone, Copy)]
+pub enum AxisScale {
+ /// Axes scale linearly
+ Linear,
+
+ /// Axes scale logarithmically
+ Logarithmic,
+}
+
+/// Contains the configuration options for the plots generated by a particular benchmark
+/// or benchmark group.
+///
+/// ```rust
+/// use self::criterion::{Bencher, Criterion, Benchmark, PlotConfiguration, AxisScale};
+///
+/// let plot_config = PlotConfiguration::default()
+/// .summary_scale(AxisScale::Logarithmic);
+///
+/// // Using Criterion::default() for simplicity; normally you'd use the macros.
+/// let mut criterion = Criterion::default();
+/// let mut benchmark_group = criterion.benchmark_group("Group name");
+/// benchmark_group.plot_config(plot_config);
+/// // Use benchmark group
+/// ```
+#[derive(Debug, Clone)]
+pub struct PlotConfiguration {
+ summary_scale: AxisScale,
+}
+
+impl Default for PlotConfiguration {
+ fn default() -> PlotConfiguration {
+ PlotConfiguration {
+ summary_scale: AxisScale::Linear,
+ }
+ }
+}
+
+impl PlotConfiguration {
+ /// Set the axis scale (linear or logarithmic) for the summary plots. Typically, you would
+ /// set this to logarithmic if benchmarking over a range of inputs which scale exponentially.
+ /// Defaults to linear.
+ pub fn summary_scale(mut self, new_scale: AxisScale) -> PlotConfiguration {
+ self.summary_scale = new_scale;
+ self
+ }
+}
+
+/// This enum allows the user to control how Criterion.rs chooses the iteration count when sampling.
+/// The default is Auto, which will choose a method automatically based on the iteration time during
+/// the warm-up phase.
+#[derive(Debug, Clone, Copy)]
+pub enum SamplingMode {
+ /// Criterion.rs should choose a sampling method automatically. This is the default, and is
+ /// recommended for most users and most benchmarks.
+ Auto,
+
+ /// Scale the iteration count in each sample linearly. This is suitable for most benchmarks,
+ /// but it tends to require many iterations which can make it very slow for very long benchmarks.
+ Linear,
+
+ /// Keep the iteration count the same for all samples. This is not recommended, as it affects
+ /// the statistics that Criterion.rs can compute. However, it requires fewer iterations than
+ /// the Linear method and therefore is more suitable for very long-running benchmarks where
+ /// benchmark execution time is more of a problem and statistical precision is less important.
+ Flat,
+}
+impl SamplingMode {
+ pub(crate) fn choose_sampling_mode(
+ &self,
+ warmup_mean_execution_time: f64,
+ sample_count: u64,
+ target_time: f64,
+ ) -> ActualSamplingMode {
+ match self {
+ SamplingMode::Linear => ActualSamplingMode::Linear,
+ SamplingMode::Flat => ActualSamplingMode::Flat,
+ SamplingMode::Auto => {
+ // Estimate execution time with linear sampling
+ let total_runs = sample_count * (sample_count + 1) / 2;
+ let d =
+ (target_time / warmup_mean_execution_time / total_runs as f64).ceil() as u64;
+ let expected_ns = total_runs as f64 * d as f64 * warmup_mean_execution_time;
+
+ if expected_ns > (2.0 * target_time) {
+ ActualSamplingMode::Flat
+ } else {
+ ActualSamplingMode::Linear
+ }
+ }
+ }
+ }
+}
+
+/// Enum to represent the sampling mode without Auto.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub(crate) enum ActualSamplingMode {
+ Linear,
+ Flat,
+}
+impl ActualSamplingMode {
+ pub(crate) fn iteration_counts(
+ &self,
+ warmup_mean_execution_time: f64,
+ sample_count: u64,
+ target_time: &Duration,
+ ) -> Vec<u64> {
+ match self {
+ ActualSamplingMode::Linear => {
+ let n = sample_count;
+ let met = warmup_mean_execution_time;
+ let m_ns = target_time.to_nanos();
+ // Solve: [d + 2*d + 3*d + ... + n*d] * met = m_ns
+ let total_runs = n * (n + 1) / 2;
+ let d = (m_ns as f64 / met / total_runs as f64).ceil() as u64;
+ let expected_ns = total_runs as f64 * d as f64 * met;
+
+ if d == 1 {
+ let recommended_sample_size =
+ ActualSamplingMode::recommend_linear_sample_size(m_ns as f64, met);
+ let actual_time = Duration::from_nanos(expected_ns as u64);
+ print!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}",
+ n, target_time, actual_time);
+
+ if recommended_sample_size != n {
+ println!(
+ ", enable flat sampling, or reduce sample count to {}.",
+ recommended_sample_size
+ );
+ } else {
+ println!("or enable flat sampling.");
+ }
+ }
+
+ (1..(n + 1) as u64).map(|a| a * d).collect::<Vec<u64>>()
+ }
+ ActualSamplingMode::Flat => {
+ let n = sample_count;
+ let met = warmup_mean_execution_time;
+ let m_ns = target_time.to_nanos() as f64;
+ let time_per_sample = m_ns / (n as f64);
+ // This is pretty simplistic; we could do something smarter to fit into the allotted time.
+ let iterations_per_sample = (time_per_sample / met).ceil() as u64;
+
+ let expected_ns = met * (iterations_per_sample * n) as f64;
+
+ if iterations_per_sample == 1 {
+ let recommended_sample_size =
+ ActualSamplingMode::recommend_flat_sample_size(m_ns, met);
+ let actual_time = Duration::from_nanos(expected_ns as u64);
+ print!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}",
+ n, target_time, actual_time);
+
+ if recommended_sample_size != n {
+ println!(", or reduce sample count to {}.", recommended_sample_size);
+ } else {
+ println!(".");
+ }
+ }
+
+ vec![iterations_per_sample; n as usize]
+ }
+ }
+ }
+
+ fn is_linear(&self) -> bool {
+ match self {
+ ActualSamplingMode::Linear => true,
+ _ => false,
+ }
+ }
+
+ fn recommend_linear_sample_size(target_time: f64, met: f64) -> u64 {
+ // Some math shows that n(n+1)/2 * d * met = target_time. d = 1, so it can be ignored.
+ // This leaves n(n+1) = (2*target_time)/met, or n^2 + n - (2*target_time)/met = 0
+ // Which can be solved with the quadratic formula. Since A and B are constant 1,
+ // this simplifies to sample_size = (-1 +- sqrt(1 - 4C))/2, where C = (2*target_time)/met.
+ // We don't care about the negative solution. Experimentation shows that this actually tends to
+ // result in twice the desired execution time (probably because of the ceil used to calculate
+ // d) so instead I use c = target_time/met.
+ let c = target_time / met;
+ let sample_size = (-1.0 + (4.0 * c).sqrt()) / 2.0;
+ let sample_size = sample_size as u64;
+
+ // Round down to the nearest 10 to give a margin and avoid excessive precision
+ let sample_size = (sample_size / 10) * 10;
+
+ // Clamp it to be at least 10, since criterion.rs doesn't allow sample sizes smaller than 10.
+ if sample_size < 10 {
+ 10
+ } else {
+ sample_size
+ }
+ }
+
+ fn recommend_flat_sample_size(target_time: f64, met: f64) -> u64 {
+ let sample_size = (target_time / met) as u64;
+
+ // Round down to the nearest 10 to give a margin and avoid excessive precision
+ let sample_size = (sample_size / 10) * 10;
+
+ // Clamp it to be at least 10, since criterion.rs doesn't allow sample sizes smaller than 10.
+ if sample_size < 10 {
+ 10
+ } else {
+ sample_size
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct SavedSample {
+ sampling_mode: ActualSamplingMode,
+ iters: Vec<f64>,
+ times: Vec<f64>,
+}
+
+/// Custom-test-framework runner. Should not be called directly.
+#[doc(hidden)]
+pub fn runner(benches: &[&dyn Fn()]) {
+ for bench in benches {
+ bench();
+ }
+ Criterion::default().configure_from_args().final_summary();
+}
diff --git a/src/macros.rs b/src/macros.rs
new file mode 100755
index 0000000..ca8bb5c
--- /dev/null
+++ b/src/macros.rs
@@ -0,0 +1,132 @@
+//! Contains macros which together define a benchmark harness that can be used
+//! in place of the standard benchmark harness. This allows the user to run
+//! Criterion.rs benchmarks with `cargo bench`.
+
+/// Macro used to define a function group for the benchmark harness; see the
+/// `criterion_main!` macro for more details.
+///
+/// This is used to define a function group; a collection of functions to call with a common
+/// Criterion configuration. Accepts two forms which can be seen below.
+///
+/// Note that the group name given here is not important, it must simply
+/// be unique. Note also that this macro is not related to the `Criterion::benchmark_group` function
+/// or the `BenchmarkGroup` type.
+///
+/// # Examples:
+///
+/// Complete form:
+///
+/// ```
+/// # #[macro_use]
+/// # extern crate criterion;
+/// # use criterion::Criterion;
+/// # fn bench_method1(c: &mut Criterion) {
+/// # }
+/// #
+/// # fn bench_method2(c: &mut Criterion) {
+/// # }
+/// #
+/// criterion_group!{
+/// name = benches;
+/// config = Criterion::default();
+/// targets = bench_method1, bench_method2
+/// }
+/// #
+/// # fn main() {}
+/// ```
+///
+/// In this form, all of the options are clearly spelled out. This expands to
+/// a function named benches, which uses the given config expression to create
+/// an instance of the Criterion struct. This is then passed by mutable
+/// reference to the targets.
+///
+/// Compact Form:
+///
+/// ```
+/// # #[macro_use]
+/// # extern crate criterion;
+/// # use criterion::Criterion;
+/// # fn bench_method1(c: &mut Criterion) {
+/// # }
+/// #
+/// # fn bench_method2(c: &mut Criterion) {
+/// # }
+/// #
+/// criterion_group!(benches, bench_method1, bench_method2);
+/// #
+/// # fn main() {}
+/// ```
+/// In this form, the first parameter is the name of the group and subsequent
+/// parameters are the target methods. The Criterion struct will be created using
+/// the `Criterion::default()` function. If you wish to customize the
+/// configuration, use the complete form and provide your own configuration
+/// function.
+#[macro_export]
+macro_rules! criterion_group {
+ (name = $name:ident; config = $config:expr; targets = $( $target:path ),+ $(,)*) => {
+ pub fn $name() {
+ let mut criterion: $crate::Criterion<_> = $config
+ .configure_from_args();
+ $(
+ $target(&mut criterion);
+ )+
+ }
+ };
+ ($name:ident, $( $target:path ),+ $(,)*) => {
+ criterion_group!{
+ name = $name;
+ config = $crate::Criterion::default();
+ targets = $( $target ),+
+ }
+ }
+}
+
+/// Macro which expands to a benchmark harness.
+///
+/// Currently, using Criterion.rs requires disabling the benchmark harness
+/// generated automatically by rustc. This can be done like so:
+///
+/// ```toml
+/// [[bench]]
+/// name = "my_bench"
+/// harness = false
+/// ```
+///
+/// In this case, `my_bench` must be a rust file inside the 'benches' directory,
+/// like so:
+///
+/// `benches/my_bench.rs`
+///
+/// Since we've disabled the default benchmark harness, we need to add our own:
+///
+/// ```ignore
+/// #[macro_use]
+/// extern crate criterion;
+/// use criterion::Criterion;
+/// fn bench_method1(c: &mut Criterion) {
+/// }
+///
+/// fn bench_method2(c: &mut Criterion) {
+/// }
+///
+/// criterion_group!(benches, bench_method1, bench_method2);
+/// criterion_main!(benches);
+/// ```
+///
+/// The `criterion_main` macro expands to a `main` function which runs all of the
+/// benchmarks in the given groups.
+///
+#[macro_export]
+macro_rules! criterion_main {
+ ( $( $group:path ),+ $(,)* ) => {
+ fn main() {
+ $(
+ $group();
+ )+
+
+ $crate::Criterion::default()
+ .configure_from_args()
+ .final_summary();
+ }
+ }
+}
diff --git a/src/macros_private.rs b/src/macros_private.rs
new file mode 100755
index 0000000..982602f
--- /dev/null
+++ b/src/macros_private.rs
@@ -0,0 +1,47 @@
+//! Private macro used for error handling.
+
+/// Logs an error, ignores an `Ok` value.
+macro_rules! log_if_err {
+ ($x:expr) => {
+ let closure = || {
+ try_else_return!($x);
+ };
+ closure();
+ };
+}
+
+/// Matches a result, returning the `Ok` value in case of success,
+/// exits the calling function otherwise.
+/// A closure which returns the return value for the function can
+/// be passed as second parameter.
+macro_rules! try_else_return {
+ ($x:expr) => {
+ try_else_return!($x, || {});
+ };
+ ($x:expr, $el:expr) => {
+ match $x {
+ Ok(x) => x,
+ Err(e) => {
+ crate::error::log_error(&e);
+ let closure = $el;
+ return closure();
+ }
+ }
+ };
+}
+
+/// Print an error message to stdout. Format is the same as println! or format!
+macro_rules! error {
+ ($($arg:tt)*) => (
+ println!("Criterion.rs ERROR: {}", &format!($($arg)*));
+ )
+}
+
+/// Print a debug message to stdout. Format is the same as println! or format!
+macro_rules! info {
+ ($($arg:tt)*) => (
+ if $crate::debug_enabled() {
+ println!("Criterion.rs DEBUG: {}", &format!($($arg)*));
+ }
+ )
+}
diff --git a/src/measurement.rs b/src/measurement.rs
new file mode 100755
index 0000000..8c6d708
--- /dev/null
+++ b/src/measurement.rs
@@ -0,0 +1,212 @@
+//! This module defines a set of traits that can be used to plug different measurements (eg.
+//! Unix's Processor Time, CPU or GPU performance counters, etc.) into Criterion.rs. It also
+//! includes the [WallTime](struct.WallTime.html) struct which defines the default wall-clock time
+//! measurement.
+
+use crate::format::short;
+use crate::DurationExt;
+use crate::Throughput;
+use std::time::{Duration, Instant};
+
+/// Trait providing functions to format measured values to string so that they can be displayed on
+/// the command line or in the reports. The functions of this trait take measured values in f64
+/// form; implementors can assume that the values are of the same scale as those produced by the
+/// associated [MeasuredValue](trait.MeasuredValue.html) (eg. if your measurement produces values in
+/// nanoseconds, the values passed to the formatter will be in nanoseconds).
+///
+/// Implementors are encouraged to format the values in a way that is intuitive for humans and
+/// uses the SI prefix system. For example, the format used by [WallTime](struct.Walltime.html)
+/// can display the value in units ranging from picoseconds to seconds depending on the magnitude
+/// of the elapsed time in nanoseconds.
+pub trait ValueFormatter {
+ /// Format the value (with appropriate unit) and return it as a string.
+ fn format_value(&self, value: f64) -> String {
+ let mut values = [value];
+ let unit = self.scale_values(value, &mut values);
+ format!("{:>6} {}", short(values[0]), unit)
+ }
+
+ /// Format the value as a throughput measurement. The value represents the measurement value;
+ /// the implementor will have to calculate bytes per second, iterations per cycle, etc.
+ fn format_throughput(&self, throughput: &Throughput, value: f64) -> String {
+ let mut values = [value];
+ let unit = self.scale_throughputs(value, throughput, &mut values);
+ format!("{:>6} {}", short(values[0]), unit)
+ }
+
+ /// Scale the given values to some appropriate unit and return the unit string.
+ ///
+ /// The given typical value should be used to choose the unit. This function may be called
+ /// multiple times with different datasets; the typical value will remain the same to ensure
+ /// that the units remain consistent within a graph. The typical value will not be NaN.
+ /// Values will not contain NaN as input, and the transformed values must not contain NaN.
+ fn scale_values(&self, typical_value: f64, values: &mut [f64]) -> &'static str;
+
+ /// Convert the given measured values into throughput numbers based on the given throughput
+ /// value, scale them to some appropriate unit, and return the unit string.
+ ///
+ /// The given typical value should be used to choose the unit. This function may be called
+ /// multiple times with different datasets; the typical value will remain the same to ensure
+ /// that the units remain consistent within a graph. The typical value will not be NaN.
+ /// Values will not contain NaN as input, and the transformed values must not contain NaN.
+ fn scale_throughputs(
+ &self,
+ typical_value: f64,
+ throughput: &Throughput,
+ values: &mut [f64],
+ ) -> &'static str;
+
+ /// Scale the values and return a unit string designed for machines.
+ ///
+ /// For example, this is used for the CSV file output. Implementations should modify the given
+ /// values slice to apply the desired scaling (if any) and return a string representing the unit
+ /// the modified values are in.
+ fn scale_for_machines(&self, values: &mut [f64]) -> &'static str;
+}
+
+/// Trait for all types which define something Criterion.rs can measure. The only measurement
+/// currently provided is [WallTime](struct.WallTime.html), but third party crates or benchmarks
+/// may define more.
+///
+/// This trait defines two core methods, `start` and `end`. `start` is called at the beginning of
+/// a measurement to produce some intermediate value (for example, the wall-clock time at the start
+/// of that set of iterations) and `end` is called at the end of the measurement with the value
+/// returned by `start`.
+///
+pub trait Measurement {
+ /// This type represents an intermediate value for the measurements. It will be produced by the
+ /// start function and passed to the end function. An example might be the wall-clock time as
+ /// of the `start` call.
+ type Intermediate;
+
+ /// This type is the measured value. An example might be the elapsed wall-clock time between the
+ /// `start` and `end` calls.
+ type Value;
+
+ /// Criterion.rs will call this before iterating the benchmark.
+ fn start(&self) -> Self::Intermediate;
+
+ /// Criterion.rs will call this after iterating the benchmark to get the measured value.
+ fn end(&self, i: Self::Intermediate) -> Self::Value;
+
+ /// Combine two values. Criterion.rs sometimes needs to perform measurements in multiple batches
+ /// of iterations, so the value from one batch must be added to the sum of the previous batches.
+ fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value;
+
+ /// Return a "zero" value for the Value type which can be added to another value.
+ fn zero(&self) -> Self::Value;
+
+ /// Converts the measured value to f64 so that it can be used in statistical analysis.
+ fn to_f64(&self, value: &Self::Value) -> f64;
+
+ /// Return a trait-object reference to the value formatter for this measurement.
+ fn formatter(&self) -> &dyn ValueFormatter;
+}
+
+pub(crate) struct DurationFormatter;
+impl DurationFormatter {
+ fn bytes_per_second(&self, bytes: f64, typical: f64, values: &mut [f64]) -> &'static str {
+ let bytes_per_second = bytes * (1e9 / typical);
+ let (denominator, unit) = if bytes_per_second < 1024.0 {
+ (1.0, " B/s")
+ } else if bytes_per_second < 1024.0 * 1024.0 {
+ (1024.0, "KiB/s")
+ } else if bytes_per_second < 1024.0 * 1024.0 * 1024.0 {
+ (1024.0 * 1024.0, "MiB/s")
+ } else {
+ (1024.0 * 1024.0 * 1024.0, "GiB/s")
+ };
+
+ for val in values {
+ let bytes_per_second = bytes * (1e9 / *val);
+ *val = bytes_per_second / denominator;
+ }
+
+ unit
+ }
+
+ fn elements_per_second(&self, elems: f64, typical: f64, values: &mut [f64]) -> &'static str {
+ let elems_per_second = elems * (1e9 / typical);
+ let (denominator, unit) = if elems_per_second < 1000.0 {
+ (1.0, " elem/s")
+ } else if elems_per_second < 1000.0 * 1000.0 {
+ (1000.0, "Kelem/s")
+ } else if elems_per_second < 1000.0 * 1000.0 * 1000.0 {
+ (1000.0 * 1000.0, "Melem/s")
+ } else {
+ (1000.0 * 1000.0 * 1000.0, "Gelem/s")
+ };
+
+ for val in values {
+ let elems_per_second = elems * (1e9 / *val);
+ *val = elems_per_second / denominator;
+ }
+
+ unit
+ }
+}
+impl ValueFormatter for DurationFormatter {
+ fn scale_throughputs(
+ &self,
+ typical: f64,
+ throughput: &Throughput,
+ values: &mut [f64],
+ ) -> &'static str {
+ match *throughput {
+ Throughput::Bytes(bytes) => self.bytes_per_second(bytes as f64, typical, values),
+ Throughput::Elements(elems) => self.elements_per_second(elems as f64, typical, values),
+ }
+ }
+
+ fn scale_values(&self, ns: f64, values: &mut [f64]) -> &'static str {
+ let (factor, unit) = if ns < 10f64.powi(0) {
+ (10f64.powi(3), "ps")
+ } else if ns < 10f64.powi(3) {
+ (10f64.powi(0), "ns")
+ } else if ns < 10f64.powi(6) {
+ (10f64.powi(-3), "us")
+ } else if ns < 10f64.powi(9) {
+ (10f64.powi(-6), "ms")
+ } else {
+ (10f64.powi(-9), "s")
+ };
+
+ for val in values {
+ *val *= factor;
+ }
+
+ unit
+ }
+
+ fn scale_for_machines(&self, _values: &mut [f64]) -> &'static str {
+ // no scaling is needed
+ "ns"
+ }
+}
+
+/// `WallTime` is the default measurement in Criterion.rs. It measures the elapsed time from the
+/// beginning of a series of iterations to the end.
+pub struct WallTime;
+impl Measurement for WallTime {
+ type Intermediate = Instant;
+ type Value = Duration;
+
+ fn start(&self) -> Self::Intermediate {
+ Instant::now()
+ }
+ fn end(&self, i: Self::Intermediate) -> Self::Value {
+ i.elapsed()
+ }
+ fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value {
+ *v1 + *v2
+ }
+ fn zero(&self) -> Self::Value {
+ Duration::from_secs(0)
+ }
+ fn to_f64(&self, val: &Self::Value) -> f64 {
+ val.to_nanos() as f64
+ }
+ fn formatter(&self) -> &dyn ValueFormatter {
+ &DurationFormatter
+ }
+}
diff --git a/src/plot/gnuplot_backend/distributions.rs b/src/plot/gnuplot_backend/distributions.rs
new file mode 100755
index 0000000..1ccbc1a
--- /dev/null
+++ b/src/plot/gnuplot_backend/distributions.rs
@@ -0,0 +1,310 @@
+use std::iter;
+use std::process::Child;
+
+use crate::stats::univariate::Sample;
+use crate::stats::Distribution;
+use criterion_plot::prelude::*;
+
+use super::*;
+use crate::estimate::Estimate;
+use crate::estimate::Statistic;
+use crate::kde;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext};
+
+fn abs_distribution(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ statistic: Statistic,
+ distribution: &Distribution<f64>,
+ estimate: &Estimate,
+ size: Option<Size>,
+) -> Child {
+ let ci = &estimate.confidence_interval;
+ let typical = ci.upper_bound;
+ let mut ci_values = [ci.lower_bound, ci.upper_bound, estimate.point_estimate];
+ let unit = formatter.scale_values(typical, &mut ci_values);
+ let (lb, ub, point) = (ci_values[0], ci_values[1], ci_values[2]);
+
+ let start = lb - (ub - lb) / 9.;
+ let end = ub + (ub - lb) / 9.;
+ let mut scaled_xs: Vec<f64> = distribution.iter().cloned().collect();
+ let _ = formatter.scale_values(typical, &mut scaled_xs);
+ let scaled_xs_sample = Sample::new(&scaled_xs);
+ let (kde_xs, ys) = kde::sweep(scaled_xs_sample, KDE_POINTS, Some((start, end)));
+
+ // interpolate between two points of the KDE sweep to find the Y position at the point estimate.
+ let n_point = kde_xs
+ .iter()
+ .position(|&x| x >= point)
+ .unwrap_or(kde_xs.len() - 1)
+ .max(1); // Must be at least the second element or this will panic
+ let slope = (ys[n_point] - ys[n_point - 1]) / (kde_xs[n_point] - kde_xs[n_point - 1]);
+ let y_point = ys[n_point - 1] + (slope * (point - kde_xs[n_point - 1]));
+
+ let zero = iter::repeat(0);
+
+ let start = kde_xs
+ .iter()
+ .enumerate()
+ .find(|&(_, &x)| x >= lb)
+ .unwrap()
+ .0;
+ let end = kde_xs
+ .iter()
+ .enumerate()
+ .rev()
+ .find(|&(_, &x)| x <= ub)
+ .unwrap()
+ .0;
+ let len = end - start;
+
+ let kde_xs_sample = Sample::new(&kde_xs);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .set(Title(format!(
+ "{}: {}",
+ gnuplot_escape(id.as_title()),
+ statistic
+ )))
+ .configure(Axis::BottomX, |a| {
+ a.set(Label(format!("Average time ({})", unit)))
+ .set(Range::Limits(kde_xs_sample.min(), kde_xs_sample.max()))
+ })
+ .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)")))
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Outside(Vertical::Top, Horizontal::Right))
+ })
+ .plot(
+ Lines {
+ x: &*kde_xs,
+ y: &*ys,
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("Bootstrap distribution"))
+ .set(LineType::Solid)
+ },
+ )
+ .plot(
+ FilledCurve {
+ x: kde_xs.iter().skip(start).take(len),
+ y1: ys.iter().skip(start),
+ y2: zero,
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("Confidence interval"))
+ .set(Opacity(0.25))
+ },
+ )
+ .plot(
+ Lines {
+ x: &[point, point],
+ y: &[0., y_point],
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("Point estimate"))
+ .set(LineType::Dash)
+ },
+ );
+
+ let path = context.report_path(id, &format!("{}.svg", statistic));
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn abs_distributions(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Vec<Child> {
+ crate::plot::REPORT_STATS
+ .iter()
+ .filter_map(|stat| {
+ measurements.distributions.get(*stat).and_then(|dist| {
+ measurements
+ .absolute_estimates
+ .get(*stat)
+ .map(|est| (*stat, dist, est))
+ })
+ })
+ .map(|(statistic, distribution, estimate)| {
+ abs_distribution(
+ id,
+ context,
+ formatter,
+ statistic,
+ distribution,
+ estimate,
+ size,
+ )
+ })
+ .collect::<Vec<_>>()
+}
+
+fn rel_distribution(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ statistic: Statistic,
+ distribution: &Distribution<f64>,
+ estimate: &Estimate,
+ noise_threshold: f64,
+ size: Option<Size>,
+) -> Child {
+ let ci = &estimate.confidence_interval;
+ let (lb, ub) = (ci.lower_bound, ci.upper_bound);
+
+ let start = lb - (ub - lb) / 9.;
+ let end = ub + (ub - lb) / 9.;
+ let (xs, ys) = kde::sweep(distribution, KDE_POINTS, Some((start, end)));
+ let xs_ = Sample::new(&xs);
+
+ // interpolate between two points of the KDE sweep to find the Y position at the point estimate.
+ let point = estimate.point_estimate;
+ let n_point = xs
+ .iter()
+ .position(|&x| x >= point)
+ .unwrap_or(ys.len() - 1)
+ .max(1);
+ let slope = (ys[n_point] - ys[n_point - 1]) / (xs[n_point] - xs[n_point - 1]);
+ let y_point = ys[n_point - 1] + (slope * (point - xs[n_point - 1]));
+
+ let one = iter::repeat(1);
+ let zero = iter::repeat(0);
+
+ let start = xs.iter().enumerate().find(|&(_, &x)| x >= lb).unwrap().0;
+ let end = xs
+ .iter()
+ .enumerate()
+ .rev()
+ .find(|&(_, &x)| x <= ub)
+ .unwrap()
+ .0;
+ let len = end - start;
+
+ let x_min = xs_.min();
+ let x_max = xs_.max();
+
+ let (fc_start, fc_end) = if noise_threshold < x_min || -noise_threshold > x_max {
+ let middle = (x_min + x_max) / 2.;
+
+ (middle, middle)
+ } else {
+ (
+ if -noise_threshold < x_min {
+ x_min
+ } else {
+ -noise_threshold
+ },
+ if noise_threshold > x_max {
+ x_max
+ } else {
+ noise_threshold
+ },
+ )
+ };
+
+ let mut figure = Figure::new();
+
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)")))
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Outside(Vertical::Top, Horizontal::Right))
+ })
+ .set(Title(format!(
+ "{}: {}",
+ gnuplot_escape(id.as_title()),
+ statistic
+ )))
+ .configure(Axis::BottomX, |a| {
+ a.set(Label("Relative change (%)"))
+ .set(Range::Limits(x_min * 100., x_max * 100.))
+ .set(ScaleFactor(100.))
+ })
+ .plot(Lines { x: &*xs, y: &*ys }, |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("Bootstrap distribution"))
+ .set(LineType::Solid)
+ })
+ .plot(
+ FilledCurve {
+ x: xs.iter().skip(start).take(len),
+ y1: ys.iter().skip(start),
+ y2: zero.clone(),
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("Confidence interval"))
+ .set(Opacity(0.25))
+ },
+ )
+ .plot(
+ Lines {
+ x: &[point, point],
+ y: &[0., y_point],
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("Point estimate"))
+ .set(LineType::Dash)
+ },
+ )
+ .plot(
+ FilledCurve {
+ x: &[fc_start, fc_end],
+ y1: one,
+ y2: zero,
+ },
+ |c| {
+ c.set(Axes::BottomXRightY)
+ .set(DARK_RED)
+ .set(Label("Noise threshold"))
+ .set(Opacity(0.1))
+ },
+ );
+
+ let path = context.report_path(id, &format!("change/{}.svg", statistic));
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn rel_distributions(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ _measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Vec<Child> {
+ crate::plot::CHANGE_STATS
+ .iter()
+ .map(|&statistic| {
+ rel_distribution(
+ id,
+ context,
+ statistic,
+ comparison.relative_distributions.get(statistic),
+ comparison.relative_estimates.get(statistic),
+ comparison.noise_threshold,
+ size,
+ )
+ })
+ .collect::<Vec<_>>()
+}
diff --git a/src/plot/gnuplot_backend/iteration_times.rs b/src/plot/gnuplot_backend/iteration_times.rs
new file mode 100755
index 0000000..4db4de8
--- /dev/null
+++ b/src/plot/gnuplot_backend/iteration_times.rs
@@ -0,0 +1,173 @@
+use std::process::Child;
+
+use criterion_plot::prelude::*;
+
+use super::*;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext};
+
+use crate::measurement::ValueFormatter;
+
+fn iteration_times_figure(
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Figure {
+ let data = &measurements.avg_times;
+ let max_avg_time = data.max();
+ let mut scaled_y: Vec<_> = data.iter().map(|(f, _)| f).collect();
+ let unit = formatter.scale_values(max_avg_time, &mut scaled_y);
+ let scaled_y = Sample::new(&scaled_y);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ a.configure(Grid::Major, |g| g.show()).set(Label("Sample"))
+ })
+ .configure(Axis::LeftY, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .set(Label(format!("Average Iteration Time ({})", unit)))
+ })
+ .plot(
+ Points {
+ x: 1..(data.len() + 1),
+ y: scaled_y.as_ref(),
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(PointSize(0.5))
+ .set(PointType::FilledCircle)
+ },
+ );
+ figure
+}
+
+pub(crate) fn iteration_times(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = iteration_times_figure(formatter, measurements, size);
+ figure.set(Title(gnuplot_escape(id.as_title())));
+ figure.configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Inside(Vertical::Top, Horizontal::Left))
+ });
+
+ let path = context.report_path(id, "iteration_times.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn iteration_times_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = iteration_times_figure(formatter, measurements, size);
+ figure.configure(Key, |k| k.hide());
+
+ let path = context.report_path(id, "iteration_times_small.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+fn iteration_times_comparison_figure(
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Figure {
+ let current_data = &measurements.avg_times;
+ let base_data = &comparison.base_avg_times;
+
+ let mut all_data: Vec<f64> = current_data.iter().map(|(f, _)| f).collect();
+ all_data.extend_from_slice(base_data);
+
+ let typical_value = Sample::new(&all_data).max();
+ let unit = formatter.scale_values(typical_value, &mut all_data);
+
+ let (scaled_current_y, scaled_base_y) = all_data.split_at(current_data.len());
+ let scaled_current_y = Sample::new(scaled_current_y);
+ let scaled_base_y = Sample::new(scaled_base_y);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ a.configure(Grid::Major, |g| g.show()).set(Label("Sample"))
+ })
+ .configure(Axis::LeftY, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .set(Label(format!("Average Iteration Time ({})", unit)))
+ })
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Inside(Vertical::Top, Horizontal::Left))
+ })
+ .plot(
+ Points {
+ x: 1..(current_data.len() + 1),
+ y: scaled_base_y.as_ref(),
+ },
+ |c| {
+ c.set(DARK_RED)
+ .set(Label("Base"))
+ .set(PointSize(0.5))
+ .set(PointType::FilledCircle)
+ },
+ )
+ .plot(
+ Points {
+ x: 1..(current_data.len() + 1),
+ y: scaled_current_y.as_ref(),
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("Current"))
+ .set(PointSize(0.5))
+ .set(PointType::FilledCircle)
+ },
+ );
+ figure
+}
+
+pub(crate) fn iteration_times_comparison(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = iteration_times_comparison_figure(formatter, measurements, comparison, size);
+ figure.set(Title(gnuplot_escape(id.as_title())));
+
+ let path = context.report_path(id, "both/iteration_times.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn iteration_times_comparison_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = iteration_times_comparison_figure(formatter, measurements, comparison, size);
+ figure.configure(Key, |k| k.hide());
+
+ let path = context.report_path(id, "relative_iteration_times_small.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
diff --git a/src/plot/gnuplot_backend/mod.rs b/src/plot/gnuplot_backend/mod.rs
new file mode 100755
index 0000000..e902d47
--- /dev/null
+++ b/src/plot/gnuplot_backend/mod.rs
@@ -0,0 +1,254 @@
+use std::iter;
+use std::path::PathBuf;
+use std::process::Child;
+
+use crate::stats::univariate::Sample;
+use criterion_plot::prelude::*;
+
+mod distributions;
+mod iteration_times;
+mod pdf;
+mod regression;
+mod summary;
+mod t_test;
+use self::distributions::*;
+use self::iteration_times::*;
+use self::pdf::*;
+use self::regression::*;
+use self::summary::*;
+use self::t_test::*;
+
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ValueType};
+use crate::stats::bivariate::Data;
+
+use super::{PlotContext, PlotData, Plotter};
+use crate::format;
+
+fn gnuplot_escape(string: &str) -> String {
+ string.replace("_", "\\_").replace("'", "''")
+}
+
+static DEFAULT_FONT: &str = "Helvetica";
+static KDE_POINTS: usize = 500;
+static SIZE: Size = Size(1280, 720);
+
+const LINEWIDTH: LineWidth = LineWidth(2.);
+const POINT_SIZE: PointSize = PointSize(0.75);
+
+const DARK_BLUE: Color = Color::Rgb(31, 120, 180);
+const DARK_ORANGE: Color = Color::Rgb(255, 127, 0);
+const DARK_RED: Color = Color::Rgb(227, 26, 28);
+
+fn debug_script(path: &PathBuf, figure: &Figure) {
+ if crate::debug_enabled() {
+ let mut script_path = path.clone();
+ script_path.set_extension("gnuplot");
+ info!("Writing gnuplot script to {:?}", script_path);
+ let result = figure.save(script_path.as_path());
+ if let Err(e) = result {
+ error!("Failed to write debug output: {}", e);
+ }
+ }
+}
+
+/// Private
+trait Append<T> {
+ /// Private
+ fn append_(self, item: T) -> Self;
+}
+
+// NB I wish this was in the standard library
+impl<T> Append<T> for Vec<T> {
+ fn append_(mut self, item: T) -> Vec<T> {
+ self.push(item);
+ self
+ }
+}
+
+#[derive(Default)]
+pub(crate) struct Gnuplot {
+ process_list: Vec<Child>,
+}
+
+impl Plotter for Gnuplot {
+ fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let size = ctx.size.map(|(w, h)| Size(w, h));
+ self.process_list.push(if ctx.is_thumbnail {
+ if let Some(cmp) = data.comparison {
+ pdf_comparison_small(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ cmp,
+ size,
+ )
+ } else {
+ pdf_small(ctx.id, ctx.context, data.formatter, data.measurements, size)
+ }
+ } else if let Some(cmp) = data.comparison {
+ pdf_comparison(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ cmp,
+ size,
+ )
+ } else {
+ pdf(ctx.id, ctx.context, data.formatter, data.measurements, size)
+ });
+ }
+
+ fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let size = ctx.size.map(|(w, h)| Size(w, h));
+ self.process_list.push(if ctx.is_thumbnail {
+ if let Some(cmp) = data.comparison {
+ let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times);
+ regression_comparison_small(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ cmp,
+ &base_data,
+ size,
+ )
+ } else {
+ regression_small(ctx.id, ctx.context, data.formatter, data.measurements, size)
+ }
+ } else if let Some(cmp) = data.comparison {
+ let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times);
+ regression_comparison(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ cmp,
+ &base_data,
+ size,
+ )
+ } else {
+ regression(ctx.id, ctx.context, data.formatter, data.measurements, size)
+ });
+ }
+
+ fn iteration_times(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let size = ctx.size.map(|(w, h)| Size(w, h));
+ self.process_list.push(if ctx.is_thumbnail {
+ if let Some(cmp) = data.comparison {
+ iteration_times_comparison_small(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ cmp,
+ size,
+ )
+ } else {
+ iteration_times_small(ctx.id, ctx.context, data.formatter, data.measurements, size)
+ }
+ } else if let Some(cmp) = data.comparison {
+ iteration_times_comparison(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ cmp,
+ size,
+ )
+ } else {
+ iteration_times(ctx.id, ctx.context, data.formatter, data.measurements, size)
+ });
+ }
+
+ fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let size = ctx.size.map(|(w, h)| Size(w, h));
+ self.process_list.extend(abs_distributions(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ size,
+ ));
+ }
+
+ fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let size = ctx.size.map(|(w, h)| Size(w, h));
+ if let Some(cmp) = data.comparison {
+ self.process_list.extend(rel_distributions(
+ ctx.id,
+ ctx.context,
+ data.measurements,
+ cmp,
+ size,
+ ));
+ } else {
+ error!("Comparison data is not provided for a relative distribution figure");
+ }
+ }
+
+ fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let size = ctx.size.map(|(w, h)| Size(w, h));
+ if let Some(cmp) = data.comparison {
+ self.process_list
+ .push(t_test(ctx.id, ctx.context, data.measurements, cmp, size));
+ } else {
+ error!("Comparison data is not provided for t_test plot");
+ }
+ }
+
+ fn line_comparison(
+ &mut self,
+ ctx: PlotContext<'_>,
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ value_type: ValueType,
+ ) {
+ let path = ctx.line_comparison_path();
+ self.process_list.push(line_comparison(
+ formatter,
+ ctx.id.as_title(),
+ all_curves,
+ &path,
+ value_type,
+ ctx.context.plot_config.summary_scale,
+ ));
+ }
+
+ fn violin(
+ &mut self,
+ ctx: PlotContext<'_>,
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ ) {
+ let violin_path = ctx.violin_path();
+
+ self.process_list.push(violin(
+ formatter,
+ ctx.id.as_title(),
+ all_curves,
+ &violin_path,
+ ctx.context.plot_config.summary_scale,
+ ));
+ }
+
+ fn wait(&mut self) {
+ let start = std::time::Instant::now();
+ let child_count = self.process_list.len();
+ for child in self.process_list.drain(..) {
+ match child.wait_with_output() {
+ Ok(ref out) if out.status.success() => {}
+ Ok(out) => error!("Error in Gnuplot: {}", String::from_utf8_lossy(&out.stderr)),
+ Err(e) => error!("Got IO error while waiting for Gnuplot to complete: {}", e),
+ }
+ }
+ let elapsed = &start.elapsed();
+ info!(
+ "Waiting for {} gnuplot processes took {}",
+ child_count,
+ format::time(crate::DurationExt::to_nanos(elapsed) as f64)
+ );
+ }
+}
diff --git a/src/plot/gnuplot_backend/pdf.rs b/src/plot/gnuplot_backend/pdf.rs
new file mode 100755
index 0000000..3ee2360
--- /dev/null
+++ b/src/plot/gnuplot_backend/pdf.rs
@@ -0,0 +1,392 @@
+use super::*;
+use crate::kde;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext};
+use std::process::Child;
+
+pub(crate) fn pdf(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Child {
+ let avg_times = &measurements.avg_times;
+ let typical = avg_times.max();
+ let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect();
+ let unit = formatter.scale_values(typical, &mut scaled_avg_times);
+ let scaled_avg_times = Sample::new(&scaled_avg_times);
+
+ let mean = scaled_avg_times.mean();
+
+ let iter_counts = measurements.iter_counts();
+ let &max_iters = iter_counts
+ .iter()
+ .max_by_key(|&&iters| iters as u64)
+ .unwrap();
+ let exponent = (max_iters.log10() / 3.).floor() as i32 * 3;
+ let y_scale = 10f64.powi(-exponent);
+
+ let y_label = if exponent == 0 {
+ "Iterations".to_owned()
+ } else {
+ format!("Iterations (x 10^{})", exponent)
+ };
+
+ let (xs, ys) = kde::sweep(&scaled_avg_times, KDE_POINTS, None);
+ let (lost, lomt, himt, hist) = avg_times.fences();
+ let mut fences = [lost, lomt, himt, hist];
+ let _ = formatter.scale_values(typical, &mut fences);
+ let [lost, lomt, himt, hist] = fences;
+
+ let vertical = &[0., max_iters];
+ let zeros = iter::repeat(0);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ let xs_ = Sample::new(&xs);
+ a.set(Label(format!("Average time ({})", unit)))
+ .set(Range::Limits(xs_.min(), xs_.max()))
+ })
+ .configure(Axis::LeftY, |a| {
+ a.set(Label(y_label))
+ .set(Range::Limits(0., max_iters * y_scale))
+ .set(ScaleFactor(y_scale))
+ })
+ .configure(Axis::RightY, |a| a.set(Label("Density (a.u.)")))
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Outside(Vertical::Top, Horizontal::Right))
+ })
+ .plot(
+ FilledCurve {
+ x: &*xs,
+ y1: &*ys,
+ y2: zeros,
+ },
+ |c| {
+ c.set(Axes::BottomXRightY)
+ .set(DARK_BLUE)
+ .set(Label("PDF"))
+ .set(Opacity(0.25))
+ },
+ )
+ .plot(
+ Lines {
+ x: &[mean, mean],
+ y: vertical,
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(LineType::Dash)
+ .set(Label("Mean"))
+ },
+ )
+ .plot(
+ Points {
+ x: avg_times
+ .iter()
+ .zip(scaled_avg_times.iter())
+ .filter_map(
+ |((_, label), t)| {
+ if label.is_outlier() {
+ None
+ } else {
+ Some(t)
+ }
+ },
+ ),
+ y: avg_times
+ .iter()
+ .zip(iter_counts.iter())
+ .filter_map(
+ |((_, label), i)| {
+ if label.is_outlier() {
+ None
+ } else {
+ Some(i)
+ }
+ },
+ ),
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("\"Clean\" sample"))
+ .set(PointType::FilledCircle)
+ .set(POINT_SIZE)
+ },
+ )
+ .plot(
+ Points {
+ x: avg_times
+ .iter()
+ .zip(scaled_avg_times.iter())
+ .filter_map(
+ |((_, label), t)| {
+ if label.is_mild() {
+ Some(t)
+ } else {
+ None
+ }
+ },
+ ),
+ y: avg_times
+ .iter()
+ .zip(iter_counts.iter())
+ .filter_map(
+ |((_, label), i)| {
+ if label.is_mild() {
+ Some(i)
+ } else {
+ None
+ }
+ },
+ ),
+ },
+ |c| {
+ c.set(DARK_ORANGE)
+ .set(Label("Mild outliers"))
+ .set(POINT_SIZE)
+ .set(PointType::FilledCircle)
+ },
+ )
+ .plot(
+ Points {
+ x: avg_times
+ .iter()
+ .zip(scaled_avg_times.iter())
+ .filter_map(
+ |((_, label), t)| {
+ if label.is_severe() {
+ Some(t)
+ } else {
+ None
+ }
+ },
+ ),
+ y: avg_times
+ .iter()
+ .zip(iter_counts.iter())
+ .filter_map(
+ |((_, label), i)| {
+ if label.is_severe() {
+ Some(i)
+ } else {
+ None
+ }
+ },
+ ),
+ },
+ |c| {
+ c.set(DARK_RED)
+ .set(Label("Severe outliers"))
+ .set(POINT_SIZE)
+ .set(PointType::FilledCircle)
+ },
+ )
+ .plot(
+ Lines {
+ x: &[lomt, lomt],
+ y: vertical,
+ },
+ |c| c.set(DARK_ORANGE).set(LINEWIDTH).set(LineType::Dash),
+ )
+ .plot(
+ Lines {
+ x: &[himt, himt],
+ y: vertical,
+ },
+ |c| c.set(DARK_ORANGE).set(LINEWIDTH).set(LineType::Dash),
+ )
+ .plot(
+ Lines {
+ x: &[lost, lost],
+ y: vertical,
+ },
+ |c| c.set(DARK_RED).set(LINEWIDTH).set(LineType::Dash),
+ )
+ .plot(
+ Lines {
+ x: &[hist, hist],
+ y: vertical,
+ },
+ |c| c.set(DARK_RED).set(LINEWIDTH).set(LineType::Dash),
+ );
+ figure.set(Title(gnuplot_escape(id.as_title())));
+
+ let path = context.report_path(id, "pdf.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn pdf_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Child {
+ let avg_times = &*measurements.avg_times;
+ let typical = avg_times.max();
+ let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect();
+ let unit = formatter.scale_values(typical, &mut scaled_avg_times);
+ let scaled_avg_times = Sample::new(&scaled_avg_times);
+ let mean = scaled_avg_times.mean();
+
+ let (xs, ys, mean_y) = kde::sweep_and_estimate(scaled_avg_times, KDE_POINTS, None, mean);
+ let xs_ = Sample::new(&xs);
+ let ys_ = Sample::new(&ys);
+
+ let y_limit = ys_.max() * 1.1;
+ let zeros = iter::repeat(0);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ a.set(Label(format!("Average time ({})", unit)))
+ .set(Range::Limits(xs_.min(), xs_.max()))
+ })
+ .configure(Axis::LeftY, |a| {
+ a.set(Label("Density (a.u.)"))
+ .set(Range::Limits(0., y_limit))
+ })
+ .configure(Axis::RightY, |a| a.hide())
+ .configure(Key, |k| k.hide())
+ .plot(
+ FilledCurve {
+ x: &*xs,
+ y1: &*ys,
+ y2: zeros,
+ },
+ |c| {
+ c.set(Axes::BottomXRightY)
+ .set(DARK_BLUE)
+ .set(Label("PDF"))
+ .set(Opacity(0.25))
+ },
+ )
+ .plot(
+ Lines {
+ x: &[mean, mean],
+ y: &[0., mean_y],
+ },
+ |c| c.set(DARK_BLUE).set(LINEWIDTH).set(Label("Mean")),
+ );
+
+ let path = context.report_path(id, "pdf_small.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+fn pdf_comparison_figure(
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Figure {
+ let base_avg_times = Sample::new(&comparison.base_avg_times);
+ let typical = base_avg_times.max().max(measurements.avg_times.max());
+ let mut scaled_base_avg_times: Vec<f64> = comparison.base_avg_times.clone();
+ let unit = formatter.scale_values(typical, &mut scaled_base_avg_times);
+ let scaled_base_avg_times = Sample::new(&scaled_base_avg_times);
+
+ let mut scaled_new_avg_times: Vec<f64> = (&measurements.avg_times as &Sample<f64>)
+ .iter()
+ .cloned()
+ .collect();
+ let _ = formatter.scale_values(typical, &mut scaled_new_avg_times);
+ let scaled_new_avg_times = Sample::new(&scaled_new_avg_times);
+
+ let base_mean = scaled_base_avg_times.mean();
+ let new_mean = scaled_new_avg_times.mean();
+
+ let (base_xs, base_ys, base_y_mean) =
+ kde::sweep_and_estimate(scaled_base_avg_times, KDE_POINTS, None, base_mean);
+ let (xs, ys, y_mean) =
+ kde::sweep_and_estimate(scaled_new_avg_times, KDE_POINTS, None, new_mean);
+
+ let zeros = iter::repeat(0);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ a.set(Label(format!("Average time ({})", unit)))
+ })
+ .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)")))
+ .configure(Axis::RightY, |a| a.hide())
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Outside(Vertical::Top, Horizontal::Right))
+ })
+ .plot(
+ FilledCurve {
+ x: &*base_xs,
+ y1: &*base_ys,
+ y2: zeros.clone(),
+ },
+ |c| c.set(DARK_RED).set(Label("Base PDF")).set(Opacity(0.5)),
+ )
+ .plot(
+ Lines {
+ x: &[base_mean, base_mean],
+ y: &[0., base_y_mean],
+ },
+ |c| c.set(DARK_RED).set(Label("Base Mean")).set(LINEWIDTH),
+ )
+ .plot(
+ FilledCurve {
+ x: &*xs,
+ y1: &*ys,
+ y2: zeros,
+ },
+ |c| c.set(DARK_BLUE).set(Label("New PDF")).set(Opacity(0.5)),
+ )
+ .plot(
+ Lines {
+ x: &[new_mean, new_mean],
+ y: &[0., y_mean],
+ },
+ |c| c.set(DARK_BLUE).set(Label("New Mean")).set(LINEWIDTH),
+ );
+ figure
+}
+
+pub(crate) fn pdf_comparison(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = pdf_comparison_figure(formatter, measurements, comparison, size);
+ figure.set(Title(gnuplot_escape(id.as_title())));
+ let path = context.report_path(id, "both/pdf.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn pdf_comparison_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = pdf_comparison_figure(formatter, measurements, comparison, size);
+ figure.configure(Key, |k| k.hide());
+ let path = context.report_path(id, "relative_pdf_small.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
diff --git a/src/plot/gnuplot_backend/regression.rs b/src/plot/gnuplot_backend/regression.rs
new file mode 100755
index 0000000..82de357
--- /dev/null
+++ b/src/plot/gnuplot_backend/regression.rs
@@ -0,0 +1,279 @@
+use std::process::Child;
+
+use crate::stats::bivariate::regression::Slope;
+use criterion_plot::prelude::*;
+
+use super::*;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext};
+use crate::stats::bivariate::Data;
+
+use crate::estimate::{ConfidenceInterval, Estimate};
+
+use crate::measurement::ValueFormatter;
+
+fn regression_figure(
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Figure {
+ let slope_estimate = measurements.absolute_estimates.slope.as_ref().unwrap();
+ let slope_dist = measurements.distributions.slope.as_ref().unwrap();
+ let (lb, ub) =
+ slope_dist.confidence_interval(slope_estimate.confidence_interval.confidence_level);
+
+ let data = &measurements.data;
+ let (max_iters, typical) = (data.x().max(), data.y().max());
+ let mut scaled_y: Vec<f64> = data.y().iter().cloned().collect();
+ let unit = formatter.scale_values(typical, &mut scaled_y);
+ let scaled_y = Sample::new(&scaled_y);
+
+ let point_estimate = Slope::fit(&measurements.data).0;
+ let mut scaled_points = [point_estimate * max_iters, lb * max_iters, ub * max_iters];
+ let _ = formatter.scale_values(typical, &mut scaled_points);
+ let [point, lb, ub] = scaled_points;
+
+ let exponent = (max_iters.log10() / 3.).floor() as i32 * 3;
+ let x_scale = 10f64.powi(-exponent);
+
+ let x_label = if exponent == 0 {
+ "Iterations".to_owned()
+ } else {
+ format!("Iterations (x 10^{})", exponent)
+ };
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .set(Label(x_label))
+ .set(ScaleFactor(x_scale))
+ })
+ .configure(Axis::LeftY, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .set(Label(format!("Total sample time ({})", unit)))
+ })
+ .plot(
+ Points {
+ x: data.x().as_ref(),
+ y: scaled_y.as_ref(),
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("Sample"))
+ .set(PointSize(0.5))
+ .set(PointType::FilledCircle)
+ },
+ )
+ .plot(
+ Lines {
+ x: &[0., max_iters],
+ y: &[0., point],
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("Linear regression"))
+ .set(LineType::Solid)
+ },
+ )
+ .plot(
+ FilledCurve {
+ x: &[0., max_iters],
+ y1: &[0., lb],
+ y2: &[0., ub],
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("Confidence interval"))
+ .set(Opacity(0.25))
+ },
+ );
+ figure
+}
+
+pub(crate) fn regression(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = regression_figure(formatter, measurements, size);
+ figure.set(Title(gnuplot_escape(id.as_title())));
+ figure.configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Inside(Vertical::Top, Horizontal::Left))
+ });
+
+ let path = context.report_path(id, "regression.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn regression_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<Size>,
+) -> Child {
+ let mut figure = regression_figure(formatter, measurements, size);
+ figure.configure(Key, |k| k.hide());
+
+ let path = context.report_path(id, "regression_small.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+fn regression_comparison_figure(
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ base_data: &Data<'_, f64, f64>,
+ size: Option<Size>,
+) -> Figure {
+ let data = &measurements.data;
+ let max_iters = base_data.x().max().max(data.x().max());
+ let typical = base_data.y().max().max(data.y().max());
+
+ let exponent = (max_iters.log10() / 3.).floor() as i32 * 3;
+ let x_scale = 10f64.powi(-exponent);
+
+ let x_label = if exponent == 0 {
+ "Iterations".to_owned()
+ } else {
+ format!("Iterations (x 10^{})", exponent)
+ };
+
+ let Estimate {
+ confidence_interval:
+ ConfidenceInterval {
+ lower_bound: base_lb,
+ upper_bound: base_ub,
+ ..
+ },
+ point_estimate: base_point,
+ ..
+ } = comparison.base_estimates.slope.as_ref().unwrap();
+
+ let Estimate {
+ confidence_interval:
+ ConfidenceInterval {
+ lower_bound: lb,
+ upper_bound: ub,
+ ..
+ },
+ point_estimate: point,
+ ..
+ } = measurements.absolute_estimates.slope.as_ref().unwrap();
+
+ let mut points = [
+ base_lb * max_iters,
+ base_point * max_iters,
+ base_ub * max_iters,
+ lb * max_iters,
+ point * max_iters,
+ ub * max_iters,
+ ];
+ let unit = formatter.scale_values(typical, &mut points);
+ let [base_lb, base_point, base_ub, lb, point, ub] = points;
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .configure(Axis::BottomX, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .set(Label(x_label))
+ .set(ScaleFactor(x_scale))
+ })
+ .configure(Axis::LeftY, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .set(Label(format!("Total sample time ({})", unit)))
+ })
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Inside(Vertical::Top, Horizontal::Left))
+ })
+ .plot(
+ FilledCurve {
+ x: &[0., max_iters],
+ y1: &[0., base_lb],
+ y2: &[0., base_ub],
+ },
+ |c| c.set(DARK_RED).set(Opacity(0.25)),
+ )
+ .plot(
+ FilledCurve {
+ x: &[0., max_iters],
+ y1: &[0., lb],
+ y2: &[0., ub],
+ },
+ |c| c.set(DARK_BLUE).set(Opacity(0.25)),
+ )
+ .plot(
+ Lines {
+ x: &[0., max_iters],
+ y: &[0., base_point],
+ },
+ |c| {
+ c.set(DARK_RED)
+ .set(LINEWIDTH)
+ .set(Label("Base sample"))
+ .set(LineType::Solid)
+ },
+ )
+ .plot(
+ Lines {
+ x: &[0., max_iters],
+ y: &[0., point],
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("New sample"))
+ .set(LineType::Solid)
+ },
+ );
+ figure
+}
+
+pub(crate) fn regression_comparison(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ base_data: &Data<'_, f64, f64>,
+ size: Option<Size>,
+) -> Child {
+ let mut figure =
+ regression_comparison_figure(formatter, measurements, comparison, base_data, size);
+ figure.set(Title(gnuplot_escape(id.as_title())));
+
+ let path = context.report_path(id, "both/regression.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
+
+pub(crate) fn regression_comparison_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ base_data: &Data<'_, f64, f64>,
+ size: Option<Size>,
+) -> Child {
+ let mut figure =
+ regression_comparison_figure(formatter, measurements, comparison, base_data, size);
+ figure.configure(Key, |k| k.hide());
+
+ let path = context.report_path(id, "relative_regression_small.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
diff --git a/src/plot/gnuplot_backend/summary.rs b/src/plot/gnuplot_backend/summary.rs
new file mode 100755
index 0000000..cdd0604
--- /dev/null
+++ b/src/plot/gnuplot_backend/summary.rs
@@ -0,0 +1,211 @@
+use super::{debug_script, gnuplot_escape};
+use super::{DARK_BLUE, DEFAULT_FONT, KDE_POINTS, LINEWIDTH, POINT_SIZE, SIZE};
+use crate::kde;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ValueType};
+use crate::stats::univariate::Sample;
+use crate::AxisScale;
+use criterion_plot::prelude::*;
+use itertools::Itertools;
+use std::cmp::Ordering;
+use std::path::{Path, PathBuf};
+use std::process::Child;
+
+const NUM_COLORS: usize = 8;
+static COMPARISON_COLORS: [Color; NUM_COLORS] = [
+ Color::Rgb(178, 34, 34),
+ Color::Rgb(46, 139, 87),
+ Color::Rgb(0, 139, 139),
+ Color::Rgb(255, 215, 0),
+ Color::Rgb(0, 0, 139),
+ Color::Rgb(220, 20, 60),
+ Color::Rgb(139, 0, 139),
+ Color::Rgb(0, 255, 127),
+];
+
+impl AxisScale {
+ fn to_gnuplot(self) -> Scale {
+ match self {
+ AxisScale::Linear => Scale::Linear,
+ AxisScale::Logarithmic => Scale::Logarithmic,
+ }
+ }
+}
+
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::explicit_counter_loop))]
+pub fn line_comparison(
+ formatter: &dyn ValueFormatter,
+ title: &str,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ path: &Path,
+ value_type: ValueType,
+ axis_scale: AxisScale,
+) -> Child {
+ let path = PathBuf::from(path);
+ let mut f = Figure::new();
+
+ let input_suffix = match value_type {
+ ValueType::Bytes => " Size (Bytes)",
+ ValueType::Elements => " Size (Elements)",
+ ValueType::Value => "",
+ };
+
+ f.set(Font(DEFAULT_FONT))
+ .set(SIZE)
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Outside(Vertical::Top, Horizontal::Right))
+ })
+ .set(Title(format!("{}: Comparison", gnuplot_escape(title))))
+ .configure(Axis::BottomX, |a| {
+ a.set(Label(format!("Input{}", input_suffix)))
+ .set(axis_scale.to_gnuplot())
+ });
+
+ let mut i = 0;
+
+ let max = all_curves
+ .iter()
+ .map(|&&(_, ref data)| Sample::new(data).mean())
+ .fold(::std::f64::NAN, f64::max);
+
+ let mut dummy = [1.0];
+ let unit = formatter.scale_values(max, &mut dummy);
+
+ f.configure(Axis::LeftY, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .configure(Grid::Minor, |g| g.hide())
+ .set(Label(format!("Average time ({})", unit)))
+ .set(axis_scale.to_gnuplot())
+ });
+
+ // This assumes the curves are sorted. It also assumes that the benchmark IDs all have numeric
+ // values or throughputs and that value is sensible (ie. not a mix of bytes and elements
+ // or whatnot)
+ for (key, group) in &all_curves.iter().group_by(|&&&(ref id, _)| &id.function_id) {
+ let mut tuples: Vec<_> = group
+ .map(|&&(ref id, ref sample)| {
+ // Unwrap is fine here because it will only fail if the assumptions above are not true
+ // ie. programmer error.
+ let x = id.as_number().unwrap();
+ let y = Sample::new(sample).mean();
+
+ (x, y)
+ })
+ .collect();
+ tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less)));
+ let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip();
+ formatter.scale_values(max, &mut ys);
+
+ let function_name = key.as_ref().map(|string| gnuplot_escape(string));
+
+ f.plot(Lines { x: &xs, y: &ys }, |c| {
+ if let Some(name) = function_name {
+ c.set(Label(name));
+ }
+ c.set(LINEWIDTH)
+ .set(LineType::Solid)
+ .set(COMPARISON_COLORS[i % NUM_COLORS])
+ })
+ .plot(Points { x: &xs, y: &ys }, |p| {
+ p.set(PointType::FilledCircle)
+ .set(POINT_SIZE)
+ .set(COMPARISON_COLORS[i % NUM_COLORS])
+ });
+
+ i += 1;
+ }
+
+ debug_script(&path, &f);
+ f.set(Output(path)).draw().unwrap()
+}
+
+pub fn violin(
+ formatter: &dyn ValueFormatter,
+ title: &str,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ path: &Path,
+ axis_scale: AxisScale,
+) -> Child {
+ let path = PathBuf::from(&path);
+ let all_curves_vec = all_curves.iter().rev().cloned().collect::<Vec<_>>();
+ let all_curves: &[&(&BenchmarkId, Vec<f64>)] = &*all_curves_vec;
+
+ let kdes = all_curves
+ .iter()
+ .map(|&&(_, ref sample)| {
+ let (x, mut y) = kde::sweep(Sample::new(sample), KDE_POINTS, None);
+ let y_max = Sample::new(&y).max();
+ for y in y.iter_mut() {
+ *y /= y_max;
+ }
+
+ (x, y)
+ })
+ .collect::<Vec<_>>();
+ let mut xs = kdes
+ .iter()
+ .flat_map(|&(ref x, _)| x.iter())
+ .filter(|&&x| x > 0.);
+ let (mut min, mut max) = {
+ let &first = xs.next().unwrap();
+ (first, first)
+ };
+ for &e in xs {
+ if e < min {
+ min = e;
+ } else if e > max {
+ max = e;
+ }
+ }
+ let mut one = [1.0];
+ // Scale the X axis units. Use the middle as a "typical value". E.g. if
+ // it is 0.002 s then this function will decide that milliseconds are an
+ // appropriate unit. It will multiple `one` by 1000, and return "ms".
+ let unit = formatter.scale_values((min + max) / 2.0, &mut one);
+
+ let tics = || (0..).map(|x| (f64::from(x)) + 0.5);
+ let size = Size(1280, 200 + (25 * all_curves.len()));
+ let mut f = Figure::new();
+ f.set(Font(DEFAULT_FONT))
+ .set(size)
+ .set(Title(format!("{}: Violin plot", gnuplot_escape(title))))
+ .configure(Axis::BottomX, |a| {
+ a.configure(Grid::Major, |g| g.show())
+ .configure(Grid::Minor, |g| g.hide())
+ .set(Label(format!("Average time ({})", unit)))
+ .set(axis_scale.to_gnuplot())
+ })
+ .configure(Axis::LeftY, |a| {
+ a.set(Label("Input"))
+ .set(Range::Limits(0., all_curves.len() as f64))
+ .set(TicLabels {
+ positions: tics(),
+ labels: all_curves
+ .iter()
+ .map(|&&(ref id, _)| gnuplot_escape(id.as_title())),
+ })
+ });
+
+ let mut is_first = true;
+ for (i, &(ref x, ref y)) in kdes.iter().enumerate() {
+ let i = i as f64 + 0.5;
+ let y1: Vec<_> = y.iter().map(|&y| i + y * 0.45).collect();
+ let y2: Vec<_> = y.iter().map(|&y| i - y * 0.45).collect();
+
+ let x: Vec<_> = x.iter().map(|&x| x * one[0]).collect();
+
+ f.plot(FilledCurve { x, y1, y2 }, |c| {
+ if is_first {
+ is_first = false;
+
+ c.set(DARK_BLUE).set(Label("PDF")).set(Opacity(0.25))
+ } else {
+ c.set(DARK_BLUE).set(Opacity(0.25))
+ }
+ });
+ }
+ debug_script(&path, &f);
+ f.set(Output(path)).draw().unwrap()
+}
diff --git a/src/plot/gnuplot_backend/t_test.rs b/src/plot/gnuplot_backend/t_test.rs
new file mode 100755
index 0000000..47b4a11
--- /dev/null
+++ b/src/plot/gnuplot_backend/t_test.rs
@@ -0,0 +1,65 @@
+use std::iter;
+use std::process::Child;
+
+use criterion_plot::prelude::*;
+
+use super::*;
+use crate::kde;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext};
+
+pub(crate) fn t_test(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ _measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<Size>,
+) -> Child {
+ let t = comparison.t_value;
+ let (xs, ys) = kde::sweep(&comparison.t_distribution, KDE_POINTS, None);
+ let zero = iter::repeat(0);
+
+ let mut figure = Figure::new();
+ figure
+ .set(Font(DEFAULT_FONT))
+ .set(size.unwrap_or(SIZE))
+ .set(Title(format!(
+ "{}: Welch t test",
+ gnuplot_escape(id.as_title())
+ )))
+ .configure(Axis::BottomX, |a| a.set(Label("t score")))
+ .configure(Axis::LeftY, |a| a.set(Label("Density")))
+ .configure(Key, |k| {
+ k.set(Justification::Left)
+ .set(Order::SampleText)
+ .set(Position::Outside(Vertical::Top, Horizontal::Right))
+ })
+ .plot(
+ FilledCurve {
+ x: &*xs,
+ y1: &*ys,
+ y2: zero,
+ },
+ |c| {
+ c.set(DARK_BLUE)
+ .set(Label("t distribution"))
+ .set(Opacity(0.25))
+ },
+ )
+ .plot(
+ Lines {
+ x: &[t, t],
+ y: &[0, 1],
+ },
+ |c| {
+ c.set(Axes::BottomXRightY)
+ .set(DARK_BLUE)
+ .set(LINEWIDTH)
+ .set(Label("t statistic"))
+ .set(LineType::Solid)
+ },
+ );
+
+ let path = context.report_path(id, "change/t-test.svg");
+ debug_script(&path, &figure);
+ figure.set(Output(path)).draw().unwrap()
+}
diff --git a/src/plot/mod.rs b/src/plot/mod.rs
new file mode 100755
index 0000000..cb836a3
--- /dev/null
+++ b/src/plot/mod.rs
@@ -0,0 +1,103 @@
+mod gnuplot_backend;
+mod plotters_backend;
+
+pub(crate) use gnuplot_backend::Gnuplot;
+pub(crate) use plotters_backend::PlottersBackend;
+
+use crate::estimate::Statistic;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext, ValueType};
+use std::path::PathBuf;
+
+const REPORT_STATS: [Statistic; 7] = [
+ Statistic::Typical,
+ Statistic::Slope,
+ Statistic::Mean,
+ Statistic::Median,
+ Statistic::MedianAbsDev,
+ Statistic::MedianAbsDev,
+ Statistic::StdDev,
+];
+const CHANGE_STATS: [Statistic; 2] = [Statistic::Mean, Statistic::Median];
+#[derive(Clone, Copy)]
+pub(crate) struct PlotContext<'a> {
+ pub(crate) id: &'a BenchmarkId,
+ pub(crate) context: &'a ReportContext,
+ pub(crate) size: Option<(usize, usize)>,
+ pub(crate) is_thumbnail: bool,
+}
+
+impl<'a> PlotContext<'a> {
+ pub fn size(mut self, s: Option<criterion_plot::Size>) -> PlotContext<'a> {
+ if let Some(s) = s {
+ self.size = Some((s.0, s.1));
+ }
+ self
+ }
+
+ pub fn thumbnail(mut self, value: bool) -> PlotContext<'a> {
+ self.is_thumbnail = value;
+ self
+ }
+
+ pub fn line_comparison_path(&self) -> PathBuf {
+ let mut path = self.context.output_directory.clone();
+ path.push(self.id.as_directory_name());
+ path.push("report");
+ path.push("lines.svg");
+ path
+ }
+
+ pub fn violin_path(&self) -> PathBuf {
+ let mut path = self.context.output_directory.clone();
+ path.push(self.id.as_directory_name());
+ path.push("report");
+ path.push("violin.svg");
+ path
+ }
+}
+
+#[derive(Clone, Copy)]
+pub(crate) struct PlotData<'a> {
+ pub(crate) formatter: &'a dyn ValueFormatter,
+ pub(crate) measurements: &'a MeasurementData<'a>,
+ pub(crate) comparison: Option<&'a ComparisonData>,
+}
+
+impl<'a> PlotData<'a> {
+ pub fn comparison(mut self, comp: &'a ComparisonData) -> PlotData<'a> {
+ self.comparison = Some(comp);
+ self
+ }
+}
+
+pub(crate) trait Plotter {
+ fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>);
+
+ fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>);
+
+ fn iteration_times(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>);
+
+ fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>);
+
+ fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>);
+
+ fn line_comparison(
+ &mut self,
+ ctx: PlotContext<'_>,
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ value_type: ValueType,
+ );
+
+ fn violin(
+ &mut self,
+ ctx: PlotContext<'_>,
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ );
+
+ fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>);
+
+ fn wait(&mut self);
+}
diff --git a/src/plot/plotters_backend/distributions.rs b/src/plot/plotters_backend/distributions.rs
new file mode 100755
index 0000000..ed4d61a
--- /dev/null
+++ b/src/plot/plotters_backend/distributions.rs
@@ -0,0 +1,309 @@
+use super::*;
+use crate::estimate::Estimate;
+use crate::estimate::Statistic;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, MeasurementData, ReportContext};
+use crate::stats::Distribution;
+
+fn abs_distribution(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ statistic: Statistic,
+ distribution: &Distribution<f64>,
+ estimate: &Estimate,
+ size: Option<(u32, u32)>,
+) {
+ let ci = &estimate.confidence_interval;
+ let typical = ci.upper_bound;
+ let mut ci_values = [ci.lower_bound, ci.upper_bound, estimate.point_estimate];
+ let unit = formatter.scale_values(typical, &mut ci_values);
+ let (lb, ub, point) = (ci_values[0], ci_values[1], ci_values[2]);
+
+ let start = lb - (ub - lb) / 9.;
+ let end = ub + (ub - lb) / 9.;
+ let mut scaled_xs: Vec<f64> = distribution.iter().cloned().collect();
+ let _ = formatter.scale_values(typical, &mut scaled_xs);
+ let scaled_xs_sample = Sample::new(&scaled_xs);
+ let (kde_xs, ys) = kde::sweep(scaled_xs_sample, KDE_POINTS, Some((start, end)));
+
+ // interpolate between two points of the KDE sweep to find the Y position at the point estimate.
+ let n_point = kde_xs
+ .iter()
+ .position(|&x| x >= point)
+ .unwrap_or(kde_xs.len() - 1)
+ .max(1); // Must be at least the second element or this will panic
+ let slope = (ys[n_point] - ys[n_point - 1]) / (kde_xs[n_point] - kde_xs[n_point - 1]);
+ let y_point = ys[n_point - 1] + (slope * (point - kde_xs[n_point - 1]));
+
+ let start = kde_xs
+ .iter()
+ .enumerate()
+ .find(|&(_, &x)| x >= lb)
+ .unwrap()
+ .0;
+ let end = kde_xs
+ .iter()
+ .enumerate()
+ .rev()
+ .find(|&(_, &x)| x <= ub)
+ .unwrap()
+ .0;
+ let len = end - start;
+
+ let kde_xs_sample = Sample::new(&kde_xs);
+
+ let path = context.report_path(id, &format!("{}.svg", statistic));
+ let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area();
+
+ let x_range = plotters::data::fitting_range(kde_xs_sample.iter());
+ let mut y_range = plotters::data::fitting_range(ys.iter());
+
+ y_range.end *= 1.1;
+
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .caption(
+ format!("{}:{}", id.as_title(), statistic),
+ (DEFAULT_FONT, 20),
+ )
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_range, y_range)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .x_desc(format!("Average time ({})", unit))
+ .y_desc("Density (a.u.)")
+ .x_label_formatter(&|&v| pretty_print_float(v, true))
+ .y_label_formatter(&|&v| pretty_print_float(v, true))
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(LineSeries::new(
+ kde_xs.iter().zip(ys.iter()).map(|(&x, &y)| (x, y)),
+ &DARK_BLUE,
+ ))
+ .unwrap()
+ .label("Bootstrap distribution")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ chart
+ .draw_series(AreaSeries::new(
+ kde_xs
+ .iter()
+ .zip(ys.iter())
+ .skip(start)
+ .take(len)
+ .map(|(&x, &y)| (x, y)),
+ 0.0,
+ DARK_BLUE.mix(0.25).filled().stroke_width(3),
+ ))
+ .unwrap()
+ .label("Confidence interval")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled())
+ });
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(point, 0.0), (point, y_point)],
+ DARK_BLUE.filled().stroke_width(3),
+ )))
+ .unwrap()
+ .label("Point estimate")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperRight)
+ .draw()
+ .unwrap();
+}
+
+pub(crate) fn abs_distributions(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<(u32, u32)>,
+) {
+ crate::plot::REPORT_STATS
+ .iter()
+ .filter_map(|stat| {
+ measurements.distributions.get(*stat).and_then(|dist| {
+ measurements
+ .absolute_estimates
+ .get(*stat)
+ .map(|est| (*stat, dist, est))
+ })
+ })
+ .for_each(|(statistic, distribution, estimate)| {
+ abs_distribution(
+ id,
+ context,
+ formatter,
+ statistic,
+ distribution,
+ estimate,
+ size,
+ )
+ })
+}
+
+fn rel_distribution(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ statistic: Statistic,
+ distribution: &Distribution<f64>,
+ estimate: &Estimate,
+ noise_threshold: f64,
+ size: Option<(u32, u32)>,
+) {
+ let ci = &estimate.confidence_interval;
+ let (lb, ub) = (ci.lower_bound, ci.upper_bound);
+
+ let start = lb - (ub - lb) / 9.;
+ let end = ub + (ub - lb) / 9.;
+ let (xs, ys) = kde::sweep(distribution, KDE_POINTS, Some((start, end)));
+ let xs_ = Sample::new(&xs);
+
+ // interpolate between two points of the KDE sweep to find the Y position at the point estimate.
+ let point = estimate.point_estimate;
+ let n_point = xs
+ .iter()
+ .position(|&x| x >= point)
+ .unwrap_or(ys.len() - 1)
+ .max(1);
+ let slope = (ys[n_point] - ys[n_point - 1]) / (xs[n_point] - xs[n_point - 1]);
+ let y_point = ys[n_point - 1] + (slope * (point - xs[n_point - 1]));
+
+ let start = xs.iter().enumerate().find(|&(_, &x)| x >= lb).unwrap().0;
+ let end = xs
+ .iter()
+ .enumerate()
+ .rev()
+ .find(|&(_, &x)| x <= ub)
+ .unwrap()
+ .0;
+ let len = end - start;
+
+ let x_min = xs_.min();
+ let x_max = xs_.max();
+
+ let (fc_start, fc_end) = if noise_threshold < x_min || -noise_threshold > x_max {
+ let middle = (x_min + x_max) / 2.;
+
+ (middle, middle)
+ } else {
+ (
+ if -noise_threshold < x_min {
+ x_min
+ } else {
+ -noise_threshold
+ },
+ if noise_threshold > x_max {
+ x_max
+ } else {
+ noise_threshold
+ },
+ )
+ };
+ let y_range = plotters::data::fitting_range(ys.iter());
+ let path = context.report_path(id, &format!("change/{}.svg", statistic));
+ let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area();
+
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .caption(
+ format!("{}:{}", id.as_title(), statistic),
+ (DEFAULT_FONT, 20),
+ )
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_min..x_max, y_range.clone())
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .x_desc("Relative change (%)")
+ .y_desc("Density (a.u.)")
+ .x_label_formatter(&|&v| pretty_print_float(v, true))
+ .y_label_formatter(&|&v| pretty_print_float(v, true))
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(LineSeries::new(
+ xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)),
+ &DARK_BLUE,
+ ))
+ .unwrap()
+ .label("Bootstrap distribution")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ chart
+ .draw_series(AreaSeries::new(
+ xs.iter()
+ .zip(ys.iter())
+ .skip(start)
+ .take(len)
+ .map(|(x, y)| (*x, *y)),
+ 0.0,
+ DARK_BLUE.mix(0.25).filled().stroke_width(3),
+ ))
+ .unwrap()
+ .label("Confidence interval")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled())
+ });
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(point, 0.0), (point, y_point)],
+ DARK_BLUE.filled().stroke_width(3),
+ )))
+ .unwrap()
+ .label("Point estimate")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ chart
+ .draw_series(std::iter::once(Rectangle::new(
+ [(fc_start, y_range.start), (fc_end, y_range.end)],
+ DARK_RED.mix(0.1).filled(),
+ )))
+ .unwrap()
+ .label("Noise threshold")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_RED.mix(0.25).filled())
+ });
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperRight)
+ .draw()
+ .unwrap();
+}
+
+pub(crate) fn rel_distributions(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ _measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<(u32, u32)>,
+) {
+ crate::plot::CHANGE_STATS.iter().for_each(|&statistic| {
+ rel_distribution(
+ id,
+ context,
+ statistic,
+ comparison.relative_distributions.get(statistic),
+ comparison.relative_estimates.get(statistic),
+ comparison.noise_threshold,
+ size,
+ )
+ });
+}
diff --git a/src/plot/plotters_backend/iteration_times.rs b/src/plot/plotters_backend/iteration_times.rs
new file mode 100755
index 0000000..a2204df
--- /dev/null
+++ b/src/plot/plotters_backend/iteration_times.rs
@@ -0,0 +1,138 @@
+use super::*;
+
+use std::path::Path;
+
+pub(crate) fn iteration_times_figure(
+ title: Option<&str>,
+ path: &Path,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<(u32, u32)>,
+) {
+ let data = &measurements.avg_times;
+ let max_avg_time = data.max();
+ let mut scaled_y: Vec<_> = data.iter().map(|(f, _)| f).collect();
+ let unit = formatter.scale_values(max_avg_time, &mut scaled_y);
+ let scaled_y = Sample::new(&scaled_y);
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(path, size).into_drawing_area();
+
+ let mut cb = ChartBuilder::on(&root_area);
+ if let Some(title) = title {
+ cb.caption(title, (DEFAULT_FONT, 20));
+ }
+
+ let x_range = (1.0)..((data.len() + 1) as f64);
+ let y_range = plotters::data::fitting_range(scaled_y.iter());
+
+ let mut chart = cb
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_range, y_range)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .y_desc(format!("Average Iteration Time ({})", unit))
+ .x_label_formatter(&|x| pretty_print_float(*x, true))
+ .line_style_2(&TRANSPARENT)
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(
+ (1..=data.len())
+ .zip(scaled_y.iter())
+ .map(|(x, y)| Circle::new((x as f64, *y), POINT_SIZE, DARK_BLUE.filled())),
+ )
+ .unwrap()
+ .label("Sample")
+ .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled()));
+
+ if title.is_some() {
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperLeft)
+ .draw()
+ .unwrap();
+ }
+}
+
+pub(crate) fn iteration_times_comparison_figure(
+ title: Option<&str>,
+ path: &Path,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<(u32, u32)>,
+) {
+ let current_data = &measurements.avg_times;
+ let base_data = &comparison.base_avg_times;
+
+ let mut all_data: Vec<f64> = current_data.iter().map(|(f, _)| f).collect();
+ all_data.extend_from_slice(base_data);
+
+ let typical_value = Sample::new(&all_data).max();
+ let unit = formatter.scale_values(typical_value, &mut all_data);
+
+ let (scaled_current_y, scaled_base_y) = all_data.split_at(current_data.len());
+ let scaled_current_y = Sample::new(scaled_current_y);
+ let scaled_base_y = Sample::new(scaled_base_y);
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(path, size).into_drawing_area();
+
+ let mut cb = ChartBuilder::on(&root_area);
+ if let Some(title) = title {
+ cb.caption(title, (DEFAULT_FONT, 20));
+ }
+
+ let max_samples = current_data.len().max(base_data.len()) as f64;
+
+ let y_range = plotters::data::fitting_range(all_data.iter());
+
+ let mut chart = cb
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(0.0..max_samples, y_range)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .y_desc(format!("Average Iteration Time ({})", unit))
+ .x_label_formatter(&|x| pretty_print_float(*x, true))
+ .line_style_2(&TRANSPARENT)
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(
+ (1..=current_data.len())
+ .zip(scaled_current_y.iter())
+ .map(|(x, y)| Circle::new((x as f64, *y), POINT_SIZE, DARK_BLUE.filled())),
+ )
+ .unwrap()
+ .label("Current")
+ .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled()));
+
+ chart
+ .draw_series(
+ (1..=base_data.len())
+ .zip(scaled_base_y.iter())
+ .map(|(x, y)| Circle::new((x as f64, *y), POINT_SIZE, DARK_RED.filled())),
+ )
+ .unwrap()
+ .label("Base")
+ .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_RED.filled()));
+
+ if title.is_some() {
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperLeft)
+ .draw()
+ .unwrap();
+ }
+}
diff --git a/src/plot/plotters_backend/mod.rs b/src/plot/plotters_backend/mod.rs
new file mode 100755
index 0000000..4cd1b18
--- /dev/null
+++ b/src/plot/plotters_backend/mod.rs
@@ -0,0 +1,232 @@
+use super::{PlotContext, PlotData, Plotter};
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ValueType};
+use plotters::data::float::pretty_print_float;
+use plotters::prelude::*;
+
+use crate::kde;
+use crate::stats::bivariate::Data;
+use crate::stats::univariate::Sample;
+
+static DEFAULT_FONT: FontFamily = FontFamily::SansSerif;
+static KDE_POINTS: usize = 500;
+static SIZE: (u32, u32) = (960, 540);
+static POINT_SIZE: u32 = 3;
+
+const DARK_BLUE: RGBColor = RGBColor(31, 120, 180);
+const DARK_ORANGE: RGBColor = RGBColor(255, 127, 0);
+const DARK_RED: RGBColor = RGBColor(227, 26, 28);
+
+mod distributions;
+mod iteration_times;
+mod pdf;
+mod regression;
+mod summary;
+mod t_test;
+
+fn convert_size(size: Option<(usize, usize)>) -> Option<(u32, u32)> {
+ if let Some((w, h)) = size {
+ return Some((w as u32, h as u32));
+ }
+ None
+}
+#[derive(Default)]
+pub struct PlottersBackend;
+
+#[allow(unused_variables)]
+impl Plotter for PlottersBackend {
+ fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ if let Some(cmp) = data.comparison {
+ let (path, title) = if ctx.is_thumbnail {
+ (
+ ctx.context.report_path(ctx.id, "relative_pdf_small.svg"),
+ None,
+ )
+ } else {
+ (
+ ctx.context.report_path(ctx.id, "both/pdf.svg"),
+ Some(ctx.id.as_title()),
+ )
+ };
+ pdf::pdf_comparison_figure(
+ path.as_ref(),
+ title,
+ data.formatter,
+ data.measurements,
+ cmp,
+ convert_size(ctx.size),
+ );
+ return;
+ }
+ if ctx.is_thumbnail {
+ pdf::pdf_small(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ convert_size(ctx.size),
+ );
+ } else {
+ pdf::pdf(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ convert_size(ctx.size),
+ );
+ }
+ }
+
+ fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let (title, path) = match (data.comparison.is_some(), ctx.is_thumbnail) {
+ (true, true) => (
+ None,
+ ctx.context
+ .report_path(ctx.id, "relative_regression_small.svg"),
+ ),
+ (true, false) => (
+ Some(ctx.id.as_title()),
+ ctx.context.report_path(ctx.id, "both/regression.svg"),
+ ),
+ (false, true) => (
+ None,
+ ctx.context.report_path(ctx.id, "regression_small.svg"),
+ ),
+ (false, false) => (
+ Some(ctx.id.as_title()),
+ ctx.context.report_path(ctx.id, "regression.svg"),
+ ),
+ };
+
+ if let Some(cmp) = data.comparison {
+ let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times);
+ regression::regression_comparison_figure(
+ title,
+ path.as_path(),
+ data.formatter,
+ data.measurements,
+ cmp,
+ &base_data,
+ convert_size(ctx.size),
+ );
+ } else {
+ regression::regression_figure(
+ title,
+ path.as_path(),
+ data.formatter,
+ data.measurements,
+ convert_size(ctx.size),
+ );
+ }
+ }
+
+ fn iteration_times(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let (title, path) = match (data.comparison.is_some(), ctx.is_thumbnail) {
+ (true, true) => (
+ None,
+ ctx.context
+ .report_path(ctx.id, "relative_iteration_times_small.svg"),
+ ),
+ (true, false) => (
+ Some(ctx.id.as_title()),
+ ctx.context.report_path(ctx.id, "both/iteration_times.svg"),
+ ),
+ (false, true) => (
+ None,
+ ctx.context.report_path(ctx.id, "iteration_times_small.svg"),
+ ),
+ (false, false) => (
+ Some(ctx.id.as_title()),
+ ctx.context.report_path(ctx.id, "iteration_times.svg"),
+ ),
+ };
+
+ if let Some(cmp) = data.comparison {
+ let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times);
+ iteration_times::iteration_times_comparison_figure(
+ title,
+ path.as_path(),
+ data.formatter,
+ data.measurements,
+ cmp,
+ convert_size(ctx.size),
+ );
+ } else {
+ iteration_times::iteration_times_figure(
+ title,
+ path.as_path(),
+ data.formatter,
+ data.measurements,
+ convert_size(ctx.size),
+ );
+ }
+ }
+
+ fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ distributions::abs_distributions(
+ ctx.id,
+ ctx.context,
+ data.formatter,
+ data.measurements,
+ convert_size(ctx.size),
+ );
+ }
+
+ fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ distributions::rel_distributions(
+ ctx.id,
+ ctx.context,
+ data.measurements,
+ data.comparison.unwrap(),
+ convert_size(ctx.size),
+ );
+ }
+
+ fn line_comparison(
+ &mut self,
+ ctx: PlotContext<'_>,
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ value_type: ValueType,
+ ) {
+ let path = ctx.line_comparison_path();
+ summary::line_comparison(
+ formatter,
+ ctx.id.as_title(),
+ all_curves,
+ &path,
+ value_type,
+ ctx.context.plot_config.summary_scale,
+ );
+ }
+
+ fn violin(
+ &mut self,
+ ctx: PlotContext<'_>,
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ ) {
+ let violin_path = ctx.violin_path();
+
+ summary::violin(
+ formatter,
+ ctx.id.as_title(),
+ all_curves,
+ &violin_path,
+ ctx.context.plot_config.summary_scale,
+ );
+ }
+
+ fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) {
+ let title = ctx.id.as_title();
+ let path = ctx.context.report_path(ctx.id, "change/t-test.svg");
+ t_test::t_test(
+ path.as_path(),
+ title,
+ data.comparison.unwrap(),
+ convert_size(ctx.size),
+ );
+ }
+
+ fn wait(&mut self) {}
+}
diff --git a/src/plot/plotters_backend/pdf.rs b/src/plot/plotters_backend/pdf.rs
new file mode 100755
index 0000000..d8f3541
--- /dev/null
+++ b/src/plot/plotters_backend/pdf.rs
@@ -0,0 +1,307 @@
+use super::*;
+use crate::measurement::ValueFormatter;
+use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext};
+use plotters::data;
+use plotters::style::RGBAColor;
+use std::path::Path;
+
+pub(crate) fn pdf_comparison_figure(
+ path: &Path,
+ title: Option<&str>,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ size: Option<(u32, u32)>,
+) {
+ let base_avg_times = Sample::new(&comparison.base_avg_times);
+ let typical = base_avg_times.max().max(measurements.avg_times.max());
+ let mut scaled_base_avg_times: Vec<f64> = comparison.base_avg_times.clone();
+ let unit = formatter.scale_values(typical, &mut scaled_base_avg_times);
+ let scaled_base_avg_times = Sample::new(&scaled_base_avg_times);
+
+ let mut scaled_new_avg_times: Vec<f64> = (&measurements.avg_times as &Sample<f64>)
+ .iter()
+ .cloned()
+ .collect();
+ let _ = formatter.scale_values(typical, &mut scaled_new_avg_times);
+ let scaled_new_avg_times = Sample::new(&scaled_new_avg_times);
+
+ let base_mean = scaled_base_avg_times.mean();
+ let new_mean = scaled_new_avg_times.mean();
+
+ let (base_xs, base_ys, base_y_mean) =
+ kde::sweep_and_estimate(scaled_base_avg_times, KDE_POINTS, None, base_mean);
+ let (xs, ys, y_mean) =
+ kde::sweep_and_estimate(scaled_new_avg_times, KDE_POINTS, None, new_mean);
+
+ let x_range = data::fitting_range(base_xs.iter().chain(xs.iter()));
+ let y_range = data::fitting_range(base_ys.iter().chain(ys.iter()));
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area();
+
+ let mut cb = ChartBuilder::on(&root_area);
+
+ if let Some(title) = title {
+ cb.caption(title, (DEFAULT_FONT, 20));
+ }
+
+ let mut chart = cb
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_range, y_range.clone())
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .y_desc("Density (a.u.)")
+ .x_desc(format!("Average Time ({})", unit))
+ .x_label_formatter(&|&x| pretty_print_float(x, true))
+ .y_label_formatter(&|&y| pretty_print_float(y, true))
+ .x_labels(5)
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(AreaSeries::new(
+ base_xs.iter().zip(base_ys.iter()).map(|(x, y)| (*x, *y)),
+ y_range.start,
+ DARK_RED.mix(0.5).filled(),
+ ))
+ .unwrap()
+ .label("Base PDF")
+ .legend(|(x, y)| Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_RED.mix(0.5).filled()));
+
+ chart
+ .draw_series(AreaSeries::new(
+ xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)),
+ y_range.start,
+ DARK_BLUE.mix(0.5).filled(),
+ ))
+ .unwrap()
+ .label("New PDF")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.5).filled())
+ });
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(base_mean, 0.0), (base_mean, base_y_mean)],
+ DARK_RED.filled().stroke_width(2),
+ )))
+ .unwrap()
+ .label("Base Mean")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_RED));
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(new_mean, 0.0), (new_mean, y_mean)],
+ DARK_BLUE.filled().stroke_width(2),
+ )))
+ .unwrap()
+ .label("New Mean")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ if title.is_some() {
+ chart.configure_series_labels().draw().unwrap();
+ }
+}
+
+pub(crate) fn pdf_small(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<(u32, u32)>,
+) {
+ let avg_times = &*measurements.avg_times;
+ let typical = avg_times.max();
+ let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect();
+ let unit = formatter.scale_values(typical, &mut scaled_avg_times);
+ let scaled_avg_times = Sample::new(&scaled_avg_times);
+ let mean = scaled_avg_times.mean();
+
+ let (xs, ys, mean_y) = kde::sweep_and_estimate(scaled_avg_times, KDE_POINTS, None, mean);
+ let xs_ = Sample::new(&xs);
+ let ys_ = Sample::new(&ys);
+
+ let y_limit = ys_.max() * 1.1;
+
+ let path = context.report_path(id, "pdf_small.svg");
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area();
+
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(xs_.min()..xs_.max(), 0.0..y_limit)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .y_desc("Density (a.u.)")
+ .x_desc(format!("Average Time ({})", unit))
+ .x_label_formatter(&|&x| pretty_print_float(x, true))
+ .y_label_formatter(&|&y| pretty_print_float(y, true))
+ .x_labels(5)
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(AreaSeries::new(
+ xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)),
+ 0.0,
+ DARK_BLUE.mix(0.25).filled(),
+ ))
+ .unwrap();
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(mean, 0.0), (mean, mean_y)],
+ DARK_BLUE.filled().stroke_width(2),
+ )))
+ .unwrap();
+}
+
+pub(crate) fn pdf(
+ id: &BenchmarkId,
+ context: &ReportContext,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<(u32, u32)>,
+) {
+ let avg_times = &measurements.avg_times;
+ let typical = avg_times.max();
+ let mut scaled_avg_times: Vec<f64> = (avg_times as &Sample<f64>).iter().cloned().collect();
+ let unit = formatter.scale_values(typical, &mut scaled_avg_times);
+ let scaled_avg_times = Sample::new(&scaled_avg_times);
+
+ let mean = scaled_avg_times.mean();
+
+ let iter_counts = measurements.iter_counts();
+ let &max_iters = iter_counts
+ .iter()
+ .max_by_key(|&&iters| iters as u64)
+ .unwrap();
+ let exponent = (max_iters.log10() / 3.).floor() as i32 * 3;
+ let y_scale = 10f64.powi(-exponent);
+
+ let y_label = if exponent == 0 {
+ "Iterations".to_owned()
+ } else {
+ format!("Iterations (x 10^{})", exponent)
+ };
+
+ let (xs, ys) = kde::sweep(&scaled_avg_times, KDE_POINTS, None);
+ let (lost, lomt, himt, hist) = avg_times.fences();
+ let mut fences = [lost, lomt, himt, hist];
+ let _ = formatter.scale_values(typical, &mut fences);
+ let [lost, lomt, himt, hist] = fences;
+
+ let path = context.report_path(id, "pdf.svg");
+
+ let xs_ = Sample::new(&xs);
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area();
+
+ let range = data::fitting_range(ys.iter());
+
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .caption(id.as_title(), (DEFAULT_FONT, 20))
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Right, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(xs_.min()..xs_.max(), 0.0..max_iters)
+ .unwrap()
+ .set_secondary_coord(xs_.min()..xs_.max(), 0.0..range.end);
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .y_desc(y_label)
+ .x_desc(format!("Average Time ({})", unit))
+ .x_label_formatter(&|&x| pretty_print_float(x, true))
+ .y_label_formatter(&|&y| pretty_print_float(y * y_scale, true))
+ .draw()
+ .unwrap();
+
+ chart
+ .configure_secondary_axes()
+ .y_desc("Density (a.u.)")
+ .x_label_formatter(&|&x| pretty_print_float(x, true))
+ .y_label_formatter(&|&y| pretty_print_float(y, true))
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_secondary_series(AreaSeries::new(
+ xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)),
+ 0.0,
+ DARK_BLUE.mix(0.5).filled(),
+ ))
+ .unwrap()
+ .label("PDF")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.5).filled())
+ });
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(mean, 0.0), (mean, max_iters)],
+ &DARK_BLUE,
+ )))
+ .unwrap()
+ .label("Mean")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ chart
+ .draw_series(vec![
+ PathElement::new(vec![(lomt, 0.0), (lomt, max_iters)], &DARK_ORANGE),
+ PathElement::new(vec![(himt, 0.0), (himt, max_iters)], &DARK_ORANGE),
+ PathElement::new(vec![(lost, 0.0), (lost, max_iters)], &DARK_RED),
+ PathElement::new(vec![(hist, 0.0), (hist, max_iters)], &DARK_RED),
+ ])
+ .unwrap();
+ use crate::stats::univariate::outliers::tukey::Label;
+
+ let mut draw_data_point_series =
+ |filter: &dyn Fn(&Label) -> bool, color: RGBAColor, name: &str| {
+ chart
+ .draw_series(
+ avg_times
+ .iter()
+ .zip(scaled_avg_times.iter())
+ .zip(iter_counts.iter())
+ .filter_map(|(((_, label), t), i)| {
+ if filter(&label) {
+ Some(Circle::new((*t, *i), POINT_SIZE, color.filled()))
+ } else {
+ None
+ }
+ }),
+ )
+ .unwrap()
+ .label(name)
+ .legend(move |(x, y)| Circle::new((x + 10, y), POINT_SIZE, color.filled()));
+ };
+
+ draw_data_point_series(
+ &|l| !l.is_outlier(),
+ DARK_BLUE.to_rgba(),
+ "\"Clean\" sample",
+ );
+ draw_data_point_series(
+ &|l| l.is_mild(),
+ RGBColor(255, 127, 0).to_rgba(),
+ "Mild outliers",
+ );
+ draw_data_point_series(&|l| l.is_severe(), DARK_RED.to_rgba(), "Severe outliers");
+ chart.configure_series_labels().draw().unwrap();
+}
diff --git a/src/plot/plotters_backend/regression.rs b/src/plot/plotters_backend/regression.rs
new file mode 100755
index 0000000..54bffa6
--- /dev/null
+++ b/src/plot/plotters_backend/regression.rs
@@ -0,0 +1,234 @@
+use super::*;
+
+use std::path::Path;
+
+use crate::estimate::{ConfidenceInterval, Estimate};
+use crate::stats::bivariate::regression::Slope;
+use crate::stats::bivariate::Data;
+
+pub(crate) fn regression_figure(
+ title: Option<&str>,
+ path: &Path,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ size: Option<(u32, u32)>,
+) {
+ let slope_estimate = measurements.absolute_estimates.slope.as_ref().unwrap();
+ let slope_dist = measurements.distributions.slope.as_ref().unwrap();
+ let (lb, ub) =
+ slope_dist.confidence_interval(slope_estimate.confidence_interval.confidence_level);
+
+ let data = &measurements.data;
+ let (max_iters, typical) = (data.x().max(), data.y().max());
+ let mut scaled_y: Vec<f64> = data.y().iter().cloned().collect();
+ let unit = formatter.scale_values(typical, &mut scaled_y);
+ let scaled_y = Sample::new(&scaled_y);
+
+ let point_estimate = Slope::fit(&measurements.data).0;
+ let mut scaled_points = [point_estimate * max_iters, lb * max_iters, ub * max_iters];
+ let _ = formatter.scale_values(typical, &mut scaled_points);
+ let [point, lb, ub] = scaled_points;
+
+ let exponent = (max_iters.log10() / 3.).floor() as i32 * 3;
+
+ let x_scale = 10f64.powi(-exponent);
+ let x_label = if exponent == 0 {
+ "Iterations".to_owned()
+ } else {
+ format!("Iterations (x 10^{})", exponent)
+ };
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(path, size).into_drawing_area();
+
+ let mut cb = ChartBuilder::on(&root_area);
+ if let Some(title) = title {
+ cb.caption(title, (DEFAULT_FONT, 20));
+ }
+
+ let x_range = plotters::data::fitting_range(data.x().iter());
+ let y_range = plotters::data::fitting_range(scaled_y.iter());
+
+ let mut chart = cb
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_range, y_range)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .x_desc(x_label)
+ .y_desc(format!("Total sample time ({})", unit))
+ .x_label_formatter(&|x| pretty_print_float(x * x_scale, true))
+ .line_style_2(&TRANSPARENT)
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(
+ data.x()
+ .iter()
+ .zip(scaled_y.iter())
+ .map(|(x, y)| Circle::new((*x, *y), POINT_SIZE, DARK_BLUE.filled())),
+ )
+ .unwrap()
+ .label("Sample")
+ .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled()));
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(0.0, 0.0), (max_iters, point)],
+ &DARK_BLUE,
+ )))
+ .unwrap()
+ .label("Linear regression")
+ .legend(|(x, y)| {
+ PathElement::new(
+ vec![(x, y), (x + 20, y)],
+ DARK_BLUE.filled().stroke_width(2),
+ )
+ });
+
+ chart
+ .draw_series(std::iter::once(Polygon::new(
+ vec![(0.0, 0.0), (max_iters, lb), (max_iters, ub)],
+ DARK_BLUE.mix(0.25).filled(),
+ )))
+ .unwrap()
+ .label("Confidence interval")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled())
+ });
+
+ if title.is_some() {
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperLeft)
+ .draw()
+ .unwrap();
+ }
+}
+
+pub(crate) fn regression_comparison_figure(
+ title: Option<&str>,
+ path: &Path,
+ formatter: &dyn ValueFormatter,
+ measurements: &MeasurementData<'_>,
+ comparison: &ComparisonData,
+ base_data: &Data<'_, f64, f64>,
+ size: Option<(u32, u32)>,
+) {
+ let data = &measurements.data;
+ let max_iters = base_data.x().max().max(data.x().max());
+ let typical = base_data.y().max().max(data.y().max());
+
+ let exponent = (max_iters.log10() / 3.).floor() as i32 * 3;
+ let x_scale = 10f64.powi(-exponent);
+
+ let x_label = if exponent == 0 {
+ "Iterations".to_owned()
+ } else {
+ format!("Iterations (x 10^{})", exponent)
+ };
+
+ let Estimate {
+ confidence_interval:
+ ConfidenceInterval {
+ lower_bound: base_lb,
+ upper_bound: base_ub,
+ ..
+ },
+ point_estimate: base_point,
+ ..
+ } = comparison.base_estimates.slope.as_ref().unwrap();
+
+ let Estimate {
+ confidence_interval:
+ ConfidenceInterval {
+ lower_bound: lb,
+ upper_bound: ub,
+ ..
+ },
+ point_estimate: point,
+ ..
+ } = measurements.absolute_estimates.slope.as_ref().unwrap();
+
+ let mut points = [
+ base_lb * max_iters,
+ base_point * max_iters,
+ base_ub * max_iters,
+ lb * max_iters,
+ point * max_iters,
+ ub * max_iters,
+ ];
+ let unit = formatter.scale_values(typical, &mut points);
+ let [base_lb, base_point, base_ub, lb, point, ub] = points;
+
+ let y_max = point.max(base_point);
+
+ let size = size.unwrap_or(SIZE);
+ let root_area = SVGBackend::new(path, size).into_drawing_area();
+
+ let mut cb = ChartBuilder::on(&root_area);
+ if let Some(title) = title {
+ cb.caption(title, (DEFAULT_FONT, 20));
+ }
+
+ let mut chart = cb
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(0.0..max_iters, 0.0..y_max)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .x_desc(x_label)
+ .y_desc(format!("Total sample time ({})", unit))
+ .x_label_formatter(&|x| pretty_print_float(x * x_scale, true))
+ .line_style_2(&TRANSPARENT)
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(vec![
+ PathElement::new(vec![(0.0, 0.0), (max_iters, base_point)], &DARK_RED).into_dyn(),
+ Polygon::new(
+ vec![(0.0, 0.0), (max_iters, base_lb), (max_iters, base_ub)],
+ DARK_RED.mix(0.25).filled(),
+ )
+ .into_dyn(),
+ ])
+ .unwrap()
+ .label("Base Sample")
+ .legend(|(x, y)| {
+ PathElement::new(vec![(x, y), (x + 20, y)], DARK_RED.filled().stroke_width(2))
+ });
+
+ chart
+ .draw_series(vec![
+ PathElement::new(vec![(0.0, 0.0), (max_iters, point)], &DARK_BLUE).into_dyn(),
+ Polygon::new(
+ vec![(0.0, 0.0), (max_iters, lb), (max_iters, ub)],
+ DARK_BLUE.mix(0.25).filled(),
+ )
+ .into_dyn(),
+ ])
+ .unwrap()
+ .label("New Sample")
+ .legend(|(x, y)| {
+ PathElement::new(
+ vec![(x, y), (x + 20, y)],
+ DARK_BLUE.filled().stroke_width(2),
+ )
+ });
+
+ if title.is_some() {
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperLeft)
+ .draw()
+ .unwrap();
+ }
+}
diff --git a/src/plot/plotters_backend/summary.rs b/src/plot/plotters_backend/summary.rs
new file mode 100755
index 0000000..dd02d0c
--- /dev/null
+++ b/src/plot/plotters_backend/summary.rs
@@ -0,0 +1,256 @@
+use super::*;
+use crate::AxisScale;
+use itertools::Itertools;
+use plotters::coord::{AsRangedCoord, Shift};
+use std::cmp::Ordering;
+use std::path::Path;
+
+const NUM_COLORS: usize = 8;
+static COMPARISON_COLORS: [RGBColor; NUM_COLORS] = [
+ RGBColor(178, 34, 34),
+ RGBColor(46, 139, 87),
+ RGBColor(0, 139, 139),
+ RGBColor(255, 215, 0),
+ RGBColor(0, 0, 139),
+ RGBColor(220, 20, 60),
+ RGBColor(139, 0, 139),
+ RGBColor(0, 255, 127),
+];
+
+pub fn line_comparison(
+ formatter: &dyn ValueFormatter,
+ title: &str,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ path: &Path,
+ value_type: ValueType,
+ axis_scale: AxisScale,
+) {
+ let (unit, series_data) = line_comparision_series_data(formatter, all_curves);
+
+ let x_range =
+ plotters::data::fitting_range(series_data.iter().map(|(_, xs, _)| xs.iter()).flatten());
+ let y_range =
+ plotters::data::fitting_range(series_data.iter().map(|(_, _, ys)| ys.iter()).flatten());
+ let root_area = SVGBackend::new(&path, SIZE)
+ .into_drawing_area()
+ .titled(&format!("{}: Comparision", title), (DEFAULT_FONT, 20))
+ .unwrap();
+
+ match axis_scale {
+ AxisScale::Linear => {
+ draw_line_comarision_figure(root_area, unit, x_range, y_range, value_type, series_data)
+ }
+ AxisScale::Logarithmic => draw_line_comarision_figure(
+ root_area,
+ unit,
+ LogRange(x_range),
+ LogRange(y_range),
+ value_type,
+ series_data,
+ ),
+ }
+}
+
+fn draw_line_comarision_figure<XR: AsRangedCoord<Value = f64>, YR: AsRangedCoord<Value = f64>>(
+ root_area: DrawingArea<SVGBackend, Shift>,
+ y_unit: &str,
+ x_range: XR,
+ y_range: YR,
+ value_type: ValueType,
+ data: Vec<(Option<&String>, Vec<f64>, Vec<f64>)>,
+) {
+ let input_suffix = match value_type {
+ ValueType::Bytes => " Size (Bytes)",
+ ValueType::Elements => " Size (Elements)",
+ ValueType::Value => "",
+ };
+
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_range, y_range)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .x_desc(format!("Input{}", input_suffix))
+ .y_desc(format!("Average time ({})", y_unit))
+ .draw()
+ .unwrap();
+
+ for (id, (name, xs, ys)) in (0..).zip(data.into_iter()) {
+ let series = chart
+ .draw_series(
+ LineSeries::new(
+ xs.into_iter().zip(ys.into_iter()),
+ COMPARISON_COLORS[id % NUM_COLORS].filled(),
+ )
+ .point_size(POINT_SIZE),
+ )
+ .unwrap();
+ if let Some(name) = name {
+ let name: &str = &*name;
+ series.label(name).legend(move |(x, y)| {
+ Rectangle::new(
+ [(x, y - 5), (x + 20, y + 5)],
+ COMPARISON_COLORS[id % NUM_COLORS].filled(),
+ )
+ });
+ }
+ }
+
+ chart
+ .configure_series_labels()
+ .position(SeriesLabelPosition::UpperLeft)
+ .draw()
+ .unwrap();
+}
+
+#[allow(clippy::type_complexity)]
+fn line_comparision_series_data<'a>(
+ formatter: &dyn ValueFormatter,
+ all_curves: &[&(&'a BenchmarkId, Vec<f64>)],
+) -> (&'static str, Vec<(Option<&'a String>, Vec<f64>, Vec<f64>)>) {
+ let max = all_curves
+ .iter()
+ .map(|&&(_, ref data)| Sample::new(data).mean())
+ .fold(::std::f64::NAN, f64::max);
+
+ let mut dummy = [1.0];
+ let unit = formatter.scale_values(max, &mut dummy);
+
+ let mut series_data = vec![];
+
+ // This assumes the curves are sorted. It also assumes that the benchmark IDs all have numeric
+ // values or throughputs and that value is sensible (ie. not a mix of bytes and elements
+ // or whatnot)
+ for (key, group) in &all_curves.iter().group_by(|&&&(ref id, _)| &id.function_id) {
+ let mut tuples: Vec<_> = group
+ .map(|&&(ref id, ref sample)| {
+ // Unwrap is fine here because it will only fail if the assumptions above are not true
+ // ie. programmer error.
+ let x = id.as_number().unwrap();
+ let y = Sample::new(sample).mean();
+
+ (x, y)
+ })
+ .collect();
+ tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less)));
+ let function_name = key.as_ref();
+ let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip();
+ formatter.scale_values(max, &mut ys);
+ series_data.push((function_name, xs, ys));
+ }
+ (unit, series_data)
+}
+
+pub fn violin(
+ formatter: &dyn ValueFormatter,
+ title: &str,
+ all_curves: &[&(&BenchmarkId, Vec<f64>)],
+ path: &Path,
+ axis_scale: AxisScale,
+) {
+ let all_curves_vec = all_curves.iter().rev().cloned().collect::<Vec<_>>();
+ let all_curves: &[&(&BenchmarkId, Vec<f64>)] = &*all_curves_vec;
+
+ let mut kdes = all_curves
+ .iter()
+ .map(|&&(ref id, ref sample)| {
+ let (x, mut y) = kde::sweep(Sample::new(sample), KDE_POINTS, None);
+ let y_max = Sample::new(&y).max();
+ for y in y.iter_mut() {
+ *y /= y_max;
+ }
+
+ (id.as_title(), x, y)
+ })
+ .collect::<Vec<_>>();
+
+ let mut xs = kdes
+ .iter()
+ .flat_map(|&(_, ref x, _)| x.iter())
+ .filter(|&&x| x > 0.);
+ let (mut min, mut max) = {
+ let &first = xs.next().unwrap();
+ (first, first)
+ };
+ for &e in xs {
+ if e < min {
+ min = e;
+ } else if e > max {
+ max = e;
+ }
+ }
+ let mut dummy = [1.0];
+ let unit = formatter.scale_values(max, &mut dummy);
+ kdes.iter_mut().for_each(|&mut (_, ref mut xs, _)| {
+ formatter.scale_values(max, xs);
+ });
+
+ let x_range = plotters::data::fitting_range(kdes.iter().map(|(_, xs, _)| xs.iter()).flatten());
+ let y_range = -0.5..all_curves.len() as f64 - 0.5;
+
+ let size = (960, 150 + (18 * all_curves.len() as u32));
+
+ let root_area = SVGBackend::new(&path, size)
+ .into_drawing_area()
+ .titled(&format!("{}: Violin plot", title), (DEFAULT_FONT, 20))
+ .unwrap();
+
+ match axis_scale {
+ AxisScale::Linear => draw_violin_figure(root_area, unit, x_range, y_range, kdes),
+ AxisScale::Logarithmic => {
+ draw_violin_figure(root_area, unit, LogRange(x_range), y_range, kdes)
+ }
+ }
+}
+
+#[allow(clippy::type_complexity)]
+fn draw_violin_figure<XR: AsRangedCoord<Value = f64>, YR: AsRangedCoord<Value = f64>>(
+ root_area: DrawingArea<SVGBackend, Shift>,
+ unit: &'static str,
+ x_range: XR,
+ y_range: YR,
+ data: Vec<(&str, Box<[f64]>, Box<[f64]>)>,
+) {
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .set_label_area_size(LabelAreaPosition::Left, (10).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_width().min(40))
+ .build_ranged(x_range, y_range)
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .y_desc("Input")
+ .x_desc(format!("Average time ({})", unit))
+ .y_label_style((DEFAULT_FONT, 10))
+ .y_label_formatter(&|v: &f64| data[v.round() as usize].0.to_string())
+ .y_labels(data.len())
+ .draw()
+ .unwrap();
+
+ for (i, (_, x, y)) in data.into_iter().enumerate() {
+ let base = i as f64;
+
+ chart
+ .draw_series(AreaSeries::new(
+ x.iter().zip(y.iter()).map(|(x, y)| (*x, base + *y / 2.0)),
+ base,
+ &DARK_BLUE.mix(0.25),
+ ))
+ .unwrap();
+
+ chart
+ .draw_series(AreaSeries::new(
+ x.iter().zip(y.iter()).map(|(x, y)| (*x, base - *y / 2.0)),
+ base,
+ &DARK_BLUE.mix(0.25),
+ ))
+ .unwrap();
+ }
+}
diff --git a/src/plot/plotters_backend/t_test.rs b/src/plot/plotters_backend/t_test.rs
new file mode 100755
index 0000000..741cc9e
--- /dev/null
+++ b/src/plot/plotters_backend/t_test.rs
@@ -0,0 +1,59 @@
+use super::*;
+use crate::report::ComparisonData;
+use std::path::Path;
+
+pub(crate) fn t_test(
+ path: &Path,
+ title: &str,
+ comparison: &ComparisonData,
+ size: Option<(u32, u32)>,
+) {
+ let t = comparison.t_value;
+ let (xs, ys) = kde::sweep(&comparison.t_distribution, KDE_POINTS, None);
+
+ let x_range = plotters::data::fitting_range(xs.iter());
+ let mut y_range = plotters::data::fitting_range(ys.iter());
+ y_range.start = 0.0;
+ y_range.end *= 1.1;
+
+ let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area();
+
+ let mut chart = ChartBuilder::on(&root_area)
+ .margin((5).percent())
+ .caption(format!("{}: Welch t test", title), (DEFAULT_FONT, 20))
+ .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
+ .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
+ .build_ranged(x_range, y_range.clone())
+ .unwrap();
+
+ chart
+ .configure_mesh()
+ .disable_mesh()
+ .y_desc("Density")
+ .x_desc("t score")
+ .draw()
+ .unwrap();
+
+ chart
+ .draw_series(AreaSeries::new(
+ xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)),
+ 0.0,
+ &DARK_BLUE.mix(0.25),
+ ))
+ .unwrap()
+ .label("t distribution")
+ .legend(|(x, y)| {
+ Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled())
+ });
+
+ chart
+ .draw_series(std::iter::once(PathElement::new(
+ vec![(t, 0.0), (t, y_range.end)],
+ DARK_BLUE.filled().stroke_width(2),
+ )))
+ .unwrap()
+ .label("t statistic")
+ .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE));
+
+ chart.configure_series_labels().draw().unwrap();
+}
diff --git a/src/profiler.rs b/src/profiler.rs
new file mode 100755
index 0000000..906af5f
--- /dev/null
+++ b/src/profiler.rs
@@ -0,0 +1,28 @@
+//! This module provides an extension trait which allows in-process profilers
+//! to be hooked into the `--profile-time` argument at compile-time. Users of
+//! out-of-process profilers such as perf don't need to do anything special.
+
+use std::path::Path;
+
+/// Extension trait for external crates to implement which provides start/stop
+/// hooks when profiling (but not when benchmarking) functions.
+pub trait Profiler {
+ /// This function is called when Criterion.rs starts profiling a particular
+ /// benchmark. It provides the stringified benchmark ID and
+ /// a path to a directory where the profiler can store its data.
+ fn start_profiling(&mut self, benchmark_id: &str, benchmark_dir: &Path);
+
+ /// This function is called after Criterion.rs stops profiling a particular
+ /// benchmark. The benchmark ID and directory are the same as in the call
+ /// to `start`, provided for convenience.
+ fn stop_profiling(&mut self, benchmark_id: &str, benchmark_dir: &Path);
+}
+
+/// Dummy profiler implementation, representing cases where the profiler is
+/// an external process (eg. perftools, etc.) which do not require start/stop
+/// hooks. This implementation does nothing and is used as the default.
+pub struct ExternalProfiler;
+impl Profiler for ExternalProfiler {
+ fn start_profiling(&mut self, _benchmark_id: &str, _benchmark_dir: &Path) {}
+ fn stop_profiling(&mut self, _benchmark_id: &str, _benchmark_dir: &Path) {}
+}
diff --git a/src/report.rs b/src/report.rs
new file mode 100755
index 0000000..216904f
--- /dev/null
+++ b/src/report.rs
@@ -0,0 +1,880 @@
+use crate::stats::bivariate::regression::Slope;
+use crate::stats::bivariate::Data;
+use crate::stats::univariate::outliers::tukey::LabeledSample;
+
+use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates};
+use crate::format;
+use crate::measurement::ValueFormatter;
+use crate::stats::univariate::Sample;
+use crate::stats::Distribution;
+use crate::{PlotConfiguration, Throughput};
+use std::cell::Cell;
+use std::cmp;
+use std::collections::HashSet;
+use std::fmt;
+use std::io::stdout;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+const MAX_DIRECTORY_NAME_LEN: usize = 64;
+const MAX_TITLE_LEN: usize = 100;
+
+pub(crate) struct ComparisonData {
+ pub p_value: f64,
+ pub t_distribution: Distribution<f64>,
+ pub t_value: f64,
+ pub relative_estimates: ChangeEstimates,
+ pub relative_distributions: ChangeDistributions,
+ pub significance_threshold: f64,
+ pub noise_threshold: f64,
+ pub base_iter_counts: Vec<f64>,
+ pub base_sample_times: Vec<f64>,
+ pub base_avg_times: Vec<f64>,
+ pub base_estimates: Estimates,
+}
+
+pub(crate) struct MeasurementData<'a> {
+ pub data: Data<'a, f64, f64>,
+ pub avg_times: LabeledSample<'a, f64>,
+ pub absolute_estimates: Estimates,
+ pub distributions: Distributions,
+ pub comparison: Option<ComparisonData>,
+ pub throughput: Option<Throughput>,
+}
+impl<'a> MeasurementData<'a> {
+ pub fn iter_counts(&self) -> &Sample<f64> {
+ self.data.x()
+ }
+
+ pub fn sample_times(&self) -> &Sample<f64> {
+ self.data.y()
+ }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum ValueType {
+ Bytes,
+ Elements,
+ Value,
+}
+
+#[derive(Clone, Serialize, Deserialize, PartialEq)]
+pub struct BenchmarkId {
+ pub group_id: String,
+ pub function_id: Option<String>,
+ pub value_str: Option<String>,
+ pub throughput: Option<Throughput>,
+ full_id: String,
+ directory_name: String,
+ title: String,
+}
+
+fn truncate_to_character_boundary(s: &mut String, max_len: usize) {
+ let mut boundary = cmp::min(max_len, s.len());
+ while !s.is_char_boundary(boundary) {
+ boundary -= 1;
+ }
+ s.truncate(boundary);
+}
+
+pub fn make_filename_safe(string: &str) -> String {
+ let mut string = string.replace(
+ &['?', '"', '/', '\\', '*', '<', '>', ':', '|', '^'][..],
+ "_",
+ );
+
+ // Truncate to last character boundary before max length...
+ truncate_to_character_boundary(&mut string, MAX_DIRECTORY_NAME_LEN);
+
+ if cfg!(target_os = "windows") {
+ {
+ string = string
+ // On Windows, spaces in the end of the filename are ignored and will be trimmed.
+ //
+ // Without trimming ourselves, creating a directory `dir ` will silently create
+ // `dir` instead, but then operations on files like `dir /file` will fail.
+ //
+ // Also note that it's important to do this *after* trimming to MAX_DIRECTORY_NAME_LEN,
+ // otherwise it can trim again to a name with a trailing space.
+ .trim_end()
+ // On Windows, file names are not case-sensitive, so lowercase everything.
+ .to_lowercase();
+ }
+ }
+
+ string
+}
+
+impl BenchmarkId {
+ pub fn new(
+ group_id: String,
+ function_id: Option<String>,
+ value_str: Option<String>,
+ throughput: Option<Throughput>,
+ ) -> BenchmarkId {
+ let full_id = match (&function_id, &value_str) {
+ (&Some(ref func), &Some(ref val)) => format!("{}/{}/{}", group_id, func, val),
+ (&Some(ref func), &None) => format!("{}/{}", group_id, func),
+ (&None, &Some(ref val)) => format!("{}/{}", group_id, val),
+ (&None, &None) => group_id.clone(),
+ };
+
+ let mut title = full_id.clone();
+ truncate_to_character_boundary(&mut title, MAX_TITLE_LEN);
+ if title != full_id {
+ title.push_str("...");
+ }
+
+ let directory_name = match (&function_id, &value_str) {
+ (&Some(ref func), &Some(ref val)) => format!(
+ "{}/{}/{}",
+ make_filename_safe(&group_id),
+ make_filename_safe(func),
+ make_filename_safe(val)
+ ),
+ (&Some(ref func), &None) => format!(
+ "{}/{}",
+ make_filename_safe(&group_id),
+ make_filename_safe(func)
+ ),
+ (&None, &Some(ref val)) => format!(
+ "{}/{}",
+ make_filename_safe(&group_id),
+ make_filename_safe(val)
+ ),
+ (&None, &None) => make_filename_safe(&group_id),
+ };
+
+ BenchmarkId {
+ group_id,
+ function_id,
+ value_str,
+ throughput,
+ full_id,
+ directory_name,
+ title,
+ }
+ }
+
+ pub fn id(&self) -> &str {
+ &self.full_id
+ }
+
+ pub fn as_title(&self) -> &str {
+ &self.title
+ }
+
+ pub fn as_directory_name(&self) -> &str {
+ &self.directory_name
+ }
+
+ pub fn as_number(&self) -> Option<f64> {
+ match self.throughput {
+ Some(Throughput::Bytes(n)) | Some(Throughput::Elements(n)) => Some(n as f64),
+ None => self
+ .value_str
+ .as_ref()
+ .and_then(|string| string.parse::<f64>().ok()),
+ }
+ }
+
+ pub fn value_type(&self) -> Option<ValueType> {
+ match self.throughput {
+ Some(Throughput::Bytes(_)) => Some(ValueType::Bytes),
+ Some(Throughput::Elements(_)) => Some(ValueType::Elements),
+ None => self
+ .value_str
+ .as_ref()
+ .and_then(|string| string.parse::<f64>().ok())
+ .map(|_| ValueType::Value),
+ }
+ }
+
+ pub fn ensure_directory_name_unique(&mut self, existing_directories: &HashSet<String>) {
+ if !existing_directories.contains(self.as_directory_name()) {
+ return;
+ }
+
+ let mut counter = 2;
+ loop {
+ let new_dir_name = format!("{}_{}", self.as_directory_name(), counter);
+ if !existing_directories.contains(&new_dir_name) {
+ self.directory_name = new_dir_name;
+ return;
+ }
+ counter += 1;
+ }
+ }
+
+ pub fn ensure_title_unique(&mut self, existing_titles: &HashSet<String>) {
+ if !existing_titles.contains(self.as_title()) {
+ return;
+ }
+
+ let mut counter = 2;
+ loop {
+ let new_title = format!("{} #{}", self.as_title(), counter);
+ if !existing_titles.contains(&new_title) {
+ self.title = new_title;
+ return;
+ }
+ counter += 1;
+ }
+ }
+}
+impl fmt::Display for BenchmarkId {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(self.as_title())
+ }
+}
+impl fmt::Debug for BenchmarkId {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fn format_opt(opt: &Option<String>) -> String {
+ match *opt {
+ Some(ref string) => format!("\"{}\"", string),
+ None => "None".to_owned(),
+ }
+ }
+
+ write!(
+ f,
+ "BenchmarkId {{ group_id: \"{}\", function_id: {}, value_str: {}, throughput: {:?} }}",
+ self.group_id,
+ format_opt(&self.function_id),
+ format_opt(&self.value_str),
+ self.throughput,
+ )
+ }
+}
+
+pub struct ReportContext {
+ pub output_directory: PathBuf,
+ pub plot_config: PlotConfiguration,
+}
+impl ReportContext {
+ pub fn report_path<P: AsRef<Path> + ?Sized>(&self, id: &BenchmarkId, file_name: &P) -> PathBuf {
+ let mut path = self.output_directory.clone();
+ path.push(id.as_directory_name());
+ path.push("report");
+ path.push(file_name);
+ path
+ }
+}
+
+pub(crate) trait Report {
+ fn test_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
+ fn test_pass(&self, _id: &BenchmarkId, _context: &ReportContext) {}
+
+ fn benchmark_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
+ fn profile(&self, _id: &BenchmarkId, _context: &ReportContext, _profile_ns: f64) {}
+ fn warmup(&self, _id: &BenchmarkId, _context: &ReportContext, _warmup_ns: f64) {}
+ fn terminated(&self, _id: &BenchmarkId, _context: &ReportContext) {}
+ fn analysis(&self, _id: &BenchmarkId, _context: &ReportContext) {}
+ fn measurement_start(
+ &self,
+ _id: &BenchmarkId,
+ _context: &ReportContext,
+ _sample_count: u64,
+ _estimate_ns: f64,
+ _iter_count: u64,
+ ) {
+ }
+ fn measurement_complete(
+ &self,
+ _id: &BenchmarkId,
+ _context: &ReportContext,
+ _measurements: &MeasurementData<'_>,
+ _formatter: &dyn ValueFormatter,
+ ) {
+ }
+ fn summarize(
+ &self,
+ _context: &ReportContext,
+ _all_ids: &[BenchmarkId],
+ _formatter: &dyn ValueFormatter,
+ ) {
+ }
+ fn final_summary(&self, _context: &ReportContext) {}
+ fn group_separator(&self) {}
+}
+
+pub(crate) struct Reports {
+ reports: Vec<Box<dyn Report>>,
+}
+impl Reports {
+ pub fn new(reports: Vec<Box<dyn Report>>) -> Reports {
+ Reports { reports }
+ }
+}
+impl Report for Reports {
+ fn test_start(&self, id: &BenchmarkId, context: &ReportContext) {
+ for report in &self.reports {
+ report.test_start(id, context);
+ }
+ }
+ fn test_pass(&self, id: &BenchmarkId, context: &ReportContext) {
+ for report in &self.reports {
+ report.test_pass(id, context);
+ }
+ }
+
+ fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext) {
+ for report in &self.reports {
+ report.benchmark_start(id, context);
+ }
+ }
+
+ fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64) {
+ for report in &self.reports {
+ report.profile(id, context, profile_ns);
+ }
+ }
+
+ fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64) {
+ for report in &self.reports {
+ report.warmup(id, context, warmup_ns);
+ }
+ }
+
+ fn terminated(&self, id: &BenchmarkId, context: &ReportContext) {
+ for report in &self.reports {
+ report.terminated(id, context);
+ }
+ }
+
+ fn analysis(&self, id: &BenchmarkId, context: &ReportContext) {
+ for report in &self.reports {
+ report.analysis(id, context);
+ }
+ }
+
+ fn measurement_start(
+ &self,
+ id: &BenchmarkId,
+ context: &ReportContext,
+ sample_count: u64,
+ estimate_ns: f64,
+ iter_count: u64,
+ ) {
+ for report in &self.reports {
+ report.measurement_start(id, context, sample_count, estimate_ns, iter_count);
+ }
+ }
+
+ fn measurement_complete(
+ &self,
+ id: &BenchmarkId,
+ context: &ReportContext,
+ measurements: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) {
+ for report in &self.reports {
+ report.measurement_complete(id, context, measurements, formatter);
+ }
+ }
+
+ fn summarize(
+ &self,
+ context: &ReportContext,
+ all_ids: &[BenchmarkId],
+ formatter: &dyn ValueFormatter,
+ ) {
+ for report in &self.reports {
+ report.summarize(context, all_ids, formatter);
+ }
+ }
+
+ fn final_summary(&self, context: &ReportContext) {
+ for report in &self.reports {
+ report.final_summary(context);
+ }
+ }
+ fn group_separator(&self) {
+ for report in &self.reports {
+ report.group_separator();
+ }
+ }
+}
+
+pub(crate) struct CliReport {
+ pub enable_text_overwrite: bool,
+ pub enable_text_coloring: bool,
+ pub verbose: bool,
+
+ last_line_len: Cell<usize>,
+}
+impl CliReport {
+ pub fn new(
+ enable_text_overwrite: bool,
+ enable_text_coloring: bool,
+ verbose: bool,
+ ) -> CliReport {
+ CliReport {
+ enable_text_overwrite,
+ enable_text_coloring,
+ verbose,
+
+ last_line_len: Cell::new(0),
+ }
+ }
+
+ fn text_overwrite(&self) {
+ if self.enable_text_overwrite {
+ print!("\r");
+ for _ in 0..self.last_line_len.get() {
+ print!(" ");
+ }
+ print!("\r");
+ }
+ }
+
+ //Passing a String is the common case here.
+ #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))]
+ fn print_overwritable(&self, s: String) {
+ if self.enable_text_overwrite {
+ self.last_line_len.set(s.len());
+ print!("{}", s);
+ stdout().flush().unwrap();
+ } else {
+ println!("{}", s);
+ }
+ }
+
+ fn green(&self, s: String) -> String {
+ if self.enable_text_coloring {
+ format!("\x1B[32m{}\x1B[39m", s)
+ } else {
+ s
+ }
+ }
+
+ fn yellow(&self, s: String) -> String {
+ if self.enable_text_coloring {
+ format!("\x1B[33m{}\x1B[39m", s)
+ } else {
+ s
+ }
+ }
+
+ fn red(&self, s: String) -> String {
+ if self.enable_text_coloring {
+ format!("\x1B[31m{}\x1B[39m", s)
+ } else {
+ s
+ }
+ }
+
+ fn bold(&self, s: String) -> String {
+ if self.enable_text_coloring {
+ format!("\x1B[1m{}\x1B[22m", s)
+ } else {
+ s
+ }
+ }
+
+ fn faint(&self, s: String) -> String {
+ if self.enable_text_coloring {
+ format!("\x1B[2m{}\x1B[22m", s)
+ } else {
+ s
+ }
+ }
+
+ pub fn outliers(&self, sample: &LabeledSample<'_, f64>) {
+ let (los, lom, _, him, his) = sample.count();
+ let noutliers = los + lom + him + his;
+ let sample_size = sample.len();
+
+ if noutliers == 0 {
+ return;
+ }
+
+ let percent = |n: usize| 100. * n as f64 / sample_size as f64;
+
+ println!(
+ "{}",
+ self.yellow(format!(
+ "Found {} outliers among {} measurements ({:.2}%)",
+ noutliers,
+ sample_size,
+ percent(noutliers)
+ ))
+ );
+
+ let print = |n, label| {
+ if n != 0 {
+ println!(" {} ({:.2}%) {}", n, percent(n), label);
+ }
+ };
+
+ print(los, "low severe");
+ print(lom, "low mild");
+ print(him, "high mild");
+ print(his, "high severe");
+ }
+}
+impl Report for CliReport {
+ fn test_start(&self, id: &BenchmarkId, _: &ReportContext) {
+ println!("Testing {}", id);
+ }
+ fn test_pass(&self, _: &BenchmarkId, _: &ReportContext) {
+ println!("Success");
+ }
+
+ fn benchmark_start(&self, id: &BenchmarkId, _: &ReportContext) {
+ self.print_overwritable(format!("Benchmarking {}", id));
+ }
+
+ fn profile(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
+ self.text_overwrite();
+ self.print_overwritable(format!(
+ "Benchmarking {}: Profiling for {}",
+ id,
+ format::time(warmup_ns)
+ ));
+ }
+
+ fn warmup(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
+ self.text_overwrite();
+ self.print_overwritable(format!(
+ "Benchmarking {}: Warming up for {}",
+ id,
+ format::time(warmup_ns)
+ ));
+ }
+
+ fn terminated(&self, id: &BenchmarkId, _: &ReportContext) {
+ self.text_overwrite();
+ println!("Benchmarking {}: Complete (Analysis Disabled)", id);
+ }
+
+ fn analysis(&self, id: &BenchmarkId, _: &ReportContext) {
+ self.text_overwrite();
+ self.print_overwritable(format!("Benchmarking {}: Analyzing", id));
+ }
+
+ fn measurement_start(
+ &self,
+ id: &BenchmarkId,
+ _: &ReportContext,
+ sample_count: u64,
+ estimate_ns: f64,
+ iter_count: u64,
+ ) {
+ self.text_overwrite();
+ let iter_string = if self.verbose {
+ format!("{} iterations", iter_count)
+ } else {
+ format::iter_count(iter_count)
+ };
+
+ self.print_overwritable(format!(
+ "Benchmarking {}: Collecting {} samples in estimated {} ({})",
+ id,
+ sample_count,
+ format::time(estimate_ns),
+ iter_string
+ ));
+ }
+
+ fn measurement_complete(
+ &self,
+ id: &BenchmarkId,
+ _: &ReportContext,
+ meas: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) {
+ self.text_overwrite();
+
+ let typical_estimate = &meas.absolute_estimates.typical();
+
+ {
+ let mut id = id.as_title().to_owned();
+
+ if id.len() > 23 {
+ println!("{}", self.green(id.clone()));
+ id.clear();
+ }
+ let id_len = id.len();
+
+ println!(
+ "{}{}time: [{} {} {}]",
+ self.green(id),
+ " ".repeat(24 - id_len),
+ self.faint(
+ formatter.format_value(typical_estimate.confidence_interval.lower_bound)
+ ),
+ self.bold(formatter.format_value(typical_estimate.point_estimate)),
+ self.faint(
+ formatter.format_value(typical_estimate.confidence_interval.upper_bound)
+ )
+ );
+ }
+
+ if let Some(ref throughput) = meas.throughput {
+ println!(
+ "{}thrpt: [{} {} {}]",
+ " ".repeat(24),
+ self.faint(formatter.format_throughput(
+ throughput,
+ typical_estimate.confidence_interval.upper_bound
+ )),
+ self.bold(formatter.format_throughput(throughput, typical_estimate.point_estimate)),
+ self.faint(formatter.format_throughput(
+ throughput,
+ typical_estimate.confidence_interval.lower_bound
+ )),
+ )
+ }
+
+ if let Some(ref comp) = meas.comparison {
+ let different_mean = comp.p_value < comp.significance_threshold;
+ let mean_est = &comp.relative_estimates.mean;
+ let point_estimate = mean_est.point_estimate;
+ let mut point_estimate_str = format::change(point_estimate, true);
+ // The change in throughput is related to the change in timing. Reducing the timing by
+ // 50% increases the througput by 100%.
+ let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
+ let mut thrpt_point_estimate_str =
+ format::change(to_thrpt_estimate(point_estimate), true);
+ let explanation_str: String;
+
+ if !different_mean {
+ explanation_str = "No change in performance detected.".to_owned();
+ } else {
+ let comparison = compare_to_threshold(&mean_est, comp.noise_threshold);
+ match comparison {
+ ComparisonResult::Improved => {
+ point_estimate_str = self.green(self.bold(point_estimate_str));
+ thrpt_point_estimate_str = self.green(self.bold(thrpt_point_estimate_str));
+ explanation_str =
+ format!("Performance has {}.", self.green("improved".to_owned()));
+ }
+ ComparisonResult::Regressed => {
+ point_estimate_str = self.red(self.bold(point_estimate_str));
+ thrpt_point_estimate_str = self.red(self.bold(thrpt_point_estimate_str));
+ explanation_str =
+ format!("Performance has {}.", self.red("regressed".to_owned()));
+ }
+ ComparisonResult::NonSignificant => {
+ explanation_str = "Change within noise threshold.".to_owned();
+ }
+ }
+ }
+
+ if meas.throughput.is_some() {
+ println!("{}change:", " ".repeat(17));
+
+ println!(
+ "{}time: [{} {} {}] (p = {:.2} {} {:.2})",
+ " ".repeat(24),
+ self.faint(format::change(
+ mean_est.confidence_interval.lower_bound,
+ true
+ )),
+ point_estimate_str,
+ self.faint(format::change(
+ mean_est.confidence_interval.upper_bound,
+ true
+ )),
+ comp.p_value,
+ if different_mean { "<" } else { ">" },
+ comp.significance_threshold
+ );
+ println!(
+ "{}thrpt: [{} {} {}]",
+ " ".repeat(24),
+ self.faint(format::change(
+ to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
+ true
+ )),
+ thrpt_point_estimate_str,
+ self.faint(format::change(
+ to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
+ true
+ )),
+ );
+ } else {
+ println!(
+ "{}change: [{} {} {}] (p = {:.2} {} {:.2})",
+ " ".repeat(24),
+ self.faint(format::change(
+ mean_est.confidence_interval.lower_bound,
+ true
+ )),
+ point_estimate_str,
+ self.faint(format::change(
+ mean_est.confidence_interval.upper_bound,
+ true
+ )),
+ comp.p_value,
+ if different_mean { "<" } else { ">" },
+ comp.significance_threshold
+ );
+ }
+
+ println!("{}{}", " ".repeat(24), explanation_str);
+ }
+
+ self.outliers(&meas.avg_times);
+
+ if self.verbose {
+ let format_short_estimate = |estimate: &Estimate| -> String {
+ format!(
+ "[{} {}]",
+ formatter.format_value(estimate.confidence_interval.lower_bound),
+ formatter.format_value(estimate.confidence_interval.upper_bound)
+ )
+ };
+
+ let data = &meas.data;
+ if let Some(slope_estimate) = meas.absolute_estimates.slope.as_ref() {
+ println!(
+ "{:<7}{} {:<15}[{:0.7} {:0.7}]",
+ "slope",
+ format_short_estimate(slope_estimate),
+ "R^2",
+ Slope(slope_estimate.confidence_interval.lower_bound).r_squared(data),
+ Slope(slope_estimate.confidence_interval.upper_bound).r_squared(data),
+ );
+ }
+ println!(
+ "{:<7}{} {:<15}{}",
+ "mean",
+ format_short_estimate(&meas.absolute_estimates.mean),
+ "std. dev.",
+ format_short_estimate(&meas.absolute_estimates.std_dev),
+ );
+ println!(
+ "{:<7}{} {:<15}{}",
+ "median",
+ format_short_estimate(&meas.absolute_estimates.median),
+ "med. abs. dev.",
+ format_short_estimate(&meas.absolute_estimates.median_abs_dev),
+ );
+ }
+ }
+
+ fn group_separator(&self) {
+ println!();
+ }
+}
+
+pub struct BencherReport;
+impl Report for BencherReport {
+ fn measurement_start(
+ &self,
+ id: &BenchmarkId,
+ _context: &ReportContext,
+ _sample_count: u64,
+ _estimate_ns: f64,
+ _iter_count: u64,
+ ) {
+ print!("test {} ... ", id);
+ }
+
+ fn measurement_complete(
+ &self,
+ _id: &BenchmarkId,
+ _: &ReportContext,
+ meas: &MeasurementData<'_>,
+ formatter: &dyn ValueFormatter,
+ ) {
+ let mut values = [
+ meas.absolute_estimates.median.point_estimate,
+ meas.absolute_estimates.std_dev.point_estimate,
+ ];
+ let unit = formatter.scale_for_machines(&mut values);
+
+ println!(
+ "bench: {:>11} {}/iter (+/- {})",
+ format::integer(values[0]),
+ unit,
+ format::integer(values[1])
+ );
+ }
+
+ fn group_separator(&self) {
+ println!();
+ }
+}
+
+enum ComparisonResult {
+ Improved,
+ Regressed,
+ NonSignificant,
+}
+
+fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
+ let ci = &estimate.confidence_interval;
+ let lb = ci.lower_bound;
+ let ub = ci.upper_bound;
+
+ if lb < -noise && ub < -noise {
+ ComparisonResult::Improved
+ } else if lb > noise && ub > noise {
+ ComparisonResult::Regressed
+ } else {
+ ComparisonResult::NonSignificant
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ #[test]
+ fn test_make_filename_safe_replaces_characters() {
+ let input = "?/\\*\"";
+ let safe = make_filename_safe(input);
+ assert_eq!("_____", &safe);
+ }
+
+ #[test]
+ fn test_make_filename_safe_truncates_long_strings() {
+ let input = "this is a very long string. it is too long to be safe as a directory name, and so it needs to be truncated. what a long string this is.";
+ let safe = make_filename_safe(input);
+ assert!(input.len() > MAX_DIRECTORY_NAME_LEN);
+ assert_eq!(&input[0..MAX_DIRECTORY_NAME_LEN], &safe);
+ }
+
+ #[test]
+ fn test_make_filename_safe_respects_character_boundaries() {
+ let input = "✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓";
+ let safe = make_filename_safe(input);
+ assert!(safe.len() < MAX_DIRECTORY_NAME_LEN);
+ }
+
+ #[test]
+ fn test_benchmark_id_make_directory_name_unique() {
+ let existing_id = BenchmarkId::new(
+ "group".to_owned(),
+ Some("function".to_owned()),
+ Some("value".to_owned()),
+ None,
+ );
+ let mut directories = HashSet::new();
+ directories.insert(existing_id.as_directory_name().to_owned());
+
+ let mut new_id = existing_id.clone();
+ new_id.ensure_directory_name_unique(&directories);
+ assert_eq!("group/function/value_2", new_id.as_directory_name());
+ directories.insert(new_id.as_directory_name().to_owned());
+
+ new_id = existing_id.clone();
+ new_id.ensure_directory_name_unique(&directories);
+ assert_eq!("group/function/value_3", new_id.as_directory_name());
+ directories.insert(new_id.as_directory_name().to_owned());
+ }
+ #[test]
+ fn test_benchmark_id_make_long_directory_name_unique() {
+ let long_name = (0..MAX_DIRECTORY_NAME_LEN).map(|_| 'a').collect::<String>();
+ let existing_id = BenchmarkId::new(long_name, None, None, None);
+ let mut directories = HashSet::new();
+ directories.insert(existing_id.as_directory_name().to_owned());
+
+ let mut new_id = existing_id.clone();
+ new_id.ensure_directory_name_unique(&directories);
+ assert_ne!(existing_id.as_directory_name(), new_id.as_directory_name());
+ }
+}
diff --git a/src/routine.rs b/src/routine.rs
new file mode 100755
index 0000000..a54cdd6
--- /dev/null
+++ b/src/routine.rs
@@ -0,0 +1,240 @@
+use crate::benchmark::BenchmarkConfig;
+use crate::connection::OutgoingMessage;
+use crate::measurement::Measurement;
+use crate::report::{BenchmarkId, ReportContext};
+use crate::{ActualSamplingMode, Bencher, Criterion, DurationExt};
+use std::marker::PhantomData;
+use std::time::Duration;
+
+/// PRIVATE
+pub(crate) trait Routine<M: Measurement, T: ?Sized> {
+ /// PRIVATE
+ fn bench(&mut self, m: &M, iters: &[u64], parameter: &T) -> Vec<f64>;
+ /// PRIVATE
+ fn warm_up(&mut self, m: &M, how_long: Duration, parameter: &T) -> (u64, u64);
+
+ /// PRIVATE
+ fn test(&mut self, m: &M, parameter: &T) {
+ self.bench(m, &[1u64], parameter);
+ }
+
+ /// Iterates the benchmarked function for a fixed length of time, but takes no measurements.
+ /// This keeps the overall benchmark suite runtime constant-ish even when running under a
+ /// profiler with an unknown amount of overhead. Since no measurements are taken, it also
+ /// reduces the amount of time the execution spends in Criterion.rs code, which should help
+ /// show the performance of the benchmarked code more clearly as well.
+ fn profile(
+ &mut self,
+ measurement: &M,
+ id: &BenchmarkId,
+ criterion: &Criterion<M>,
+ report_context: &ReportContext,
+ time: Duration,
+ parameter: &T,
+ ) {
+ criterion
+ .report
+ .profile(id, report_context, time.to_nanos() as f64);
+
+ let mut profile_path = report_context.output_directory.clone();
+ if (*crate::CARGO_CRITERION_CONNECTION).is_some() {
+ // If connected to cargo-criterion, generate a cargo-criterion-style path.
+ // This is kind of a hack.
+ profile_path.push("profile");
+ profile_path.push(id.as_directory_name());
+ } else {
+ profile_path.push(id.as_directory_name());
+ profile_path.push("profile");
+ }
+ criterion
+ .profiler
+ .borrow_mut()
+ .start_profiling(id.id(), &profile_path);
+
+ let time = time.to_nanos();
+
+ // TODO: Some profilers will show the two batches of iterations as
+ // being different code-paths even though they aren't really.
+
+ // Get the warmup time for one second
+ let (wu_elapsed, wu_iters) = self.warm_up(measurement, Duration::from_secs(1), parameter);
+ if wu_elapsed < time {
+ // Initial guess for the mean execution time
+ let met = wu_elapsed as f64 / wu_iters as f64;
+
+ // Guess how many iterations will be required for the remaining time
+ let remaining = (time - wu_elapsed) as f64;
+
+ let iters = remaining / met;
+ let iters = iters as u64;
+
+ self.bench(measurement, &[iters], parameter);
+ }
+
+ criterion
+ .profiler
+ .borrow_mut()
+ .stop_profiling(id.id(), &profile_path);
+
+ criterion.report.terminated(id, report_context);
+ }
+
+ fn sample(
+ &mut self,
+ measurement: &M,
+ id: &BenchmarkId,
+ config: &BenchmarkConfig,
+ criterion: &Criterion<M>,
+ report_context: &ReportContext,
+ parameter: &T,
+ ) -> (ActualSamplingMode, Box<[f64]>, Box<[f64]>) {
+ let wu = config.warm_up_time;
+ let m_ns = config.measurement_time.to_nanos();
+
+ criterion
+ .report
+ .warmup(id, report_context, wu.to_nanos() as f64);
+
+ if let Some(conn) = &criterion.connection {
+ conn.send(&OutgoingMessage::Warmup {
+ id: id.into(),
+ nanos: wu.to_nanos() as f64,
+ })
+ .unwrap();
+ }
+
+ let (wu_elapsed, wu_iters) = self.warm_up(measurement, wu, parameter);
+ if crate::debug_enabled() {
+ println!(
+ "\nCompleted {} iterations in {} nanoseconds, estimated execution time is {} ns",
+ wu_iters,
+ wu_elapsed,
+ wu_elapsed as f64 / wu_iters as f64
+ );
+ }
+
+ // Initial guess for the mean execution time
+ let met = wu_elapsed as f64 / wu_iters as f64;
+
+ let n = config.sample_size as u64;
+
+ let actual_sampling_mode = config
+ .sampling_mode
+ .choose_sampling_mode(met, n, m_ns as f64);
+
+ let m_iters = actual_sampling_mode.iteration_counts(met, n, &config.measurement_time);
+
+ let expected_ns = m_iters
+ .iter()
+ .copied()
+ .map(|count| count as f64 * met)
+ .sum();
+
+ criterion.report.measurement_start(
+ id,
+ report_context,
+ n,
+ expected_ns,
+ m_iters.iter().sum(),
+ );
+
+ if let Some(conn) = &criterion.connection {
+ conn.send(&OutgoingMessage::MeasurementStart {
+ id: id.into(),
+ sample_count: n,
+ estimate_ns: expected_ns,
+ iter_count: m_iters.iter().sum(),
+ })
+ .unwrap();
+ }
+
+ let m_elapsed = self.bench(measurement, &m_iters, parameter);
+
+ let m_iters_f: Vec<f64> = m_iters.iter().map(|&x| x as f64).collect();
+
+ (
+ actual_sampling_mode,
+ m_iters_f.into_boxed_slice(),
+ m_elapsed.into_boxed_slice(),
+ )
+ }
+}
+
+pub struct Function<M: Measurement, F, T>
+where
+ F: FnMut(&mut Bencher<'_, M>, &T),
+ T: ?Sized,
+{
+ f: F,
+ // TODO: Is there some way to remove these?
+ _phantom: PhantomData<T>,
+ _phamtom2: PhantomData<M>,
+}
+impl<M: Measurement, F, T> Function<M, F, T>
+where
+ F: FnMut(&mut Bencher<'_, M>, &T),
+ T: ?Sized,
+{
+ pub fn new(f: F) -> Function<M, F, T> {
+ Function {
+ f,
+ _phantom: PhantomData,
+ _phamtom2: PhantomData,
+ }
+ }
+}
+
+impl<M: Measurement, F, T> Routine<M, T> for Function<M, F, T>
+where
+ F: FnMut(&mut Bencher<'_, M>, &T),
+ T: ?Sized,
+{
+ fn bench(&mut self, m: &M, iters: &[u64], parameter: &T) -> Vec<f64> {
+ let f = &mut self.f;
+
+ let mut b = Bencher {
+ iterated: false,
+ iters: 0,
+ value: m.zero(),
+ measurement: m,
+ elapsed_time: Duration::from_millis(0),
+ };
+
+ iters
+ .iter()
+ .map(|iters| {
+ b.iters = *iters;
+ (*f)(&mut b, parameter);
+ b.assert_iterated();
+ m.to_f64(&b.value)
+ })
+ .collect()
+ }
+
+ fn warm_up(&mut self, m: &M, how_long: Duration, parameter: &T) -> (u64, u64) {
+ let f = &mut self.f;
+ let mut b = Bencher {
+ iterated: false,
+ iters: 1,
+ value: m.zero(),
+ measurement: m,
+ elapsed_time: Duration::from_millis(0),
+ };
+
+ let mut total_iters = 0;
+ let mut elapsed_time = Duration::from_millis(0);
+ loop {
+ (*f)(&mut b, parameter);
+
+ b.assert_iterated();
+
+ total_iters += b.iters;
+ elapsed_time += b.elapsed_time;
+ if elapsed_time > how_long {
+ return (elapsed_time.to_nanos(), total_iters);
+ }
+
+ b.iters *= 2;
+ }
+ }
+}
diff --git a/src/stats/bivariate/bootstrap.rs b/src/stats/bivariate/bootstrap.rs
new file mode 100755
index 0000000..8fe8ede
--- /dev/null
+++ b/src/stats/bivariate/bootstrap.rs
@@ -0,0 +1,83 @@
+#[cfg(test)]
+macro_rules! test {
+ ($ty:ident) => {
+ mod $ty {
+ use quickcheck::TestResult;
+ use quickcheck::quickcheck;
+ use approx::relative_eq;
+
+ use crate::stats::bivariate::regression::Slope;
+ use crate::stats::bivariate::Data;
+
+ quickcheck! {
+ fn means(size: usize, start: usize,
+ offset: usize, nresamples: usize) -> TestResult {
+ if let Some(x) = crate::stats::test::vec::<$ty>(size, start) {
+ let y = crate::stats::test::vec::<$ty>(size + offset, start + offset).unwrap();
+ let data = Data::new(&x[start..], &y[start+offset..]);
+
+ let (x_means, y_means) = if nresamples > 0 {
+ data.bootstrap(nresamples, |d| (d.x().mean(), d.y().mean()))
+ } else {
+ return TestResult::discard();
+ };
+
+ let x_min = data.x().min();
+ let x_max = data.x().max();
+ let y_min = data.y().min();
+ let y_max = data.y().max();
+
+ TestResult::from_bool(
+ // Computed the correct number of resamples
+ x_means.len() == nresamples &&
+ y_means.len() == nresamples &&
+ // No uninitialized values
+ x_means.iter().all(|&x| {
+ (x > x_min || relative_eq!(x, x_min)) &&
+ (x < x_max || relative_eq!(x, x_max))
+ }) &&
+ y_means.iter().all(|&y| {
+ (y > y_min || relative_eq!(y, y_min)) &&
+ (y < y_max || relative_eq!(y, y_max))
+ })
+ )
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+
+ quickcheck! {
+ fn slope(size: usize, start: usize,
+ offset: usize, nresamples: usize) -> TestResult {
+ if let Some(x) = crate::stats::test::vec::<$ty>(size, start) {
+ let y = crate::stats::test::vec::<$ty>(size + offset, start + offset).unwrap();
+ let data = Data::new(&x[start..], &y[start+offset..]);
+
+ let slopes = if nresamples > 0 {
+ data.bootstrap(nresamples, |d| (Slope::fit(&d),)).0
+ } else {
+ return TestResult::discard();
+ };
+
+ TestResult::from_bool(
+ // Computed the correct number of resamples
+ slopes.len() == nresamples &&
+ // No uninitialized values
+ slopes.iter().all(|s| s.0 > 0.)
+ )
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+
+ }
+ };
+}
+
+#[cfg(test)]
+mod test {
+ test!(f32);
+ test!(f64);
+}
diff --git a/src/stats/bivariate/mod.rs b/src/stats/bivariate/mod.rs
new file mode 100755
index 0000000..b233b60
--- /dev/null
+++ b/src/stats/bivariate/mod.rs
@@ -0,0 +1,131 @@
+//! Bivariate analysis
+
+mod bootstrap;
+pub mod regression;
+mod resamples;
+
+use crate::stats::bivariate::resamples::Resamples;
+use crate::stats::float::Float;
+use crate::stats::tuple::{Tuple, TupledDistributionsBuilder};
+use crate::stats::univariate::Sample;
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
+
+/// Bivariate `(X, Y)` data
+///
+/// Invariants:
+///
+/// - No `NaN`s in the data
+/// - At least two data points in the set
+pub struct Data<'a, X, Y>(&'a [X], &'a [Y]);
+
+impl<'a, X, Y> Copy for Data<'a, X, Y> {}
+
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::expl_impl_clone_on_copy))]
+impl<'a, X, Y> Clone for Data<'a, X, Y> {
+ fn clone(&self) -> Data<'a, X, Y> {
+ *self
+ }
+}
+
+impl<'a, X, Y> Data<'a, X, Y> {
+ /// Returns the length of the data set
+ pub fn len(&self) -> usize {
+ self.0.len()
+ }
+
+ /// Iterate over the data set
+ pub fn iter(&self) -> Pairs<'a, X, Y> {
+ Pairs {
+ data: *self,
+ state: 0,
+ }
+ }
+}
+
+impl<'a, X, Y> Data<'a, X, Y>
+where
+ X: Float,
+ Y: Float,
+{
+ /// Creates a new data set from two existing slices
+ pub fn new(xs: &'a [X], ys: &'a [Y]) -> Data<'a, X, Y> {
+ assert!(
+ xs.len() == ys.len()
+ && xs.len() > 1
+ && xs.iter().all(|x| !x.is_nan())
+ && ys.iter().all(|y| !y.is_nan())
+ );
+
+ Data(xs, ys)
+ }
+
+ // TODO Remove the `T` parameter in favor of `S::Output`
+ /// Returns the bootstrap distributions of the parameters estimated by the `statistic`
+ ///
+ /// - Multi-threaded
+ /// - Time: `O(nresamples)`
+ /// - Memory: `O(nresamples)`
+ pub fn bootstrap<T, S>(&self, nresamples: usize, statistic: S) -> T::Distributions
+ where
+ S: Fn(Data<X, Y>) -> T + Sync,
+ T: Tuple + Send,
+ T::Distributions: Send,
+ T::Builder: Send,
+ {
+ (0..nresamples)
+ .into_par_iter()
+ .map_init(
+ || Resamples::new(*self),
+ |resamples, _| statistic(resamples.next()),
+ )
+ .fold(
+ || T::Builder::new(0),
+ |mut sub_distributions, sample| {
+ sub_distributions.push(sample);
+ sub_distributions
+ },
+ )
+ .reduce(
+ || T::Builder::new(0),
+ |mut a, mut b| {
+ a.extend(&mut b);
+ a
+ },
+ )
+ .complete()
+ }
+
+ /// Returns a view into the `X` data
+ pub fn x(&self) -> &'a Sample<X> {
+ Sample::new(&self.0)
+ }
+
+ /// Returns a view into the `Y` data
+ pub fn y(&self) -> &'a Sample<Y> {
+ Sample::new(&self.1)
+ }
+}
+
+/// Iterator over `Data`
+pub struct Pairs<'a, X: 'a, Y: 'a> {
+ data: Data<'a, X, Y>,
+ state: usize,
+}
+
+impl<'a, X, Y> Iterator for Pairs<'a, X, Y> {
+ type Item = (&'a X, &'a Y);
+
+ fn next(&mut self) -> Option<(&'a X, &'a Y)> {
+ if self.state < self.data.len() {
+ let i = self.state;
+ self.state += 1;
+
+ // This is safe because i will always be < self.data.{0,1}.len()
+ debug_assert!(i < self.data.0.len());
+ debug_assert!(i < self.data.1.len());
+ unsafe { Some((self.data.0.get_unchecked(i), self.data.1.get_unchecked(i))) }
+ } else {
+ None
+ }
+ }
+}
diff --git a/src/stats/bivariate/regression.rs b/src/stats/bivariate/regression.rs
new file mode 100755
index 0000000..f09443f
--- /dev/null
+++ b/src/stats/bivariate/regression.rs
@@ -0,0 +1,53 @@
+//! Regression analysis
+
+use crate::stats::bivariate::Data;
+use crate::stats::float::Float;
+
+/// A straight line that passes through the origin `y = m * x`
+#[derive(Clone, Copy)]
+pub struct Slope<A>(pub A)
+where
+ A: Float;
+
+impl<A> Slope<A>
+where
+ A: Float,
+{
+ /// Fits the data to a straight line that passes through the origin using ordinary least
+ /// squares
+ ///
+ /// - Time: `O(length)`
+ pub fn fit(data: &Data<'_, A, A>) -> Slope<A> {
+ let xs = data.0;
+ let ys = data.1;
+
+ let xy = crate::stats::dot(xs, ys);
+ let x2 = crate::stats::dot(xs, xs);
+
+ Slope(xy / x2)
+ }
+
+ /// Computes the goodness of fit (coefficient of determination) for this data set
+ ///
+ /// - Time: `O(length)`
+ pub fn r_squared(&self, data: &Data<'_, A, A>) -> A {
+ let _0 = A::cast(0);
+ let _1 = A::cast(1);
+ let m = self.0;
+ let xs = data.0;
+ let ys = data.1;
+
+ let n = A::cast(xs.len());
+ let y_bar = crate::stats::sum(ys) / n;
+
+ let mut ss_res = _0;
+ let mut ss_tot = _0;
+
+ for (&x, &y) in data.iter() {
+ ss_res = ss_res + (y - m * x).powi(2);
+ ss_tot = ss_res + (y - y_bar).powi(2);
+ }
+
+ _1 - ss_res / ss_tot
+ }
+}
diff --git a/src/stats/bivariate/resamples.rs b/src/stats/bivariate/resamples.rs
new file mode 100755
index 0000000..e254dc7
--- /dev/null
+++ b/src/stats/bivariate/resamples.rs
@@ -0,0 +1,61 @@
+use crate::stats::bivariate::Data;
+use crate::stats::float::Float;
+use crate::stats::rand_util::{new_rng, Rng};
+
+pub struct Resamples<'a, X, Y>
+where
+ X: 'a + Float,
+ Y: 'a + Float,
+{
+ rng: Rng,
+ data: (&'a [X], &'a [Y]),
+ stage: Option<(Vec<X>, Vec<Y>)>,
+}
+
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))]
+impl<'a, X, Y> Resamples<'a, X, Y>
+where
+ X: 'a + Float,
+ Y: 'a + Float,
+{
+ pub fn new(data: Data<'a, X, Y>) -> Resamples<'a, X, Y> {
+ Resamples {
+ rng: new_rng(),
+ data: (data.x(), data.y()),
+ stage: None,
+ }
+ }
+
+ pub fn next(&mut self) -> Data<'_, X, Y> {
+ let n = self.data.0.len();
+
+ match self.stage {
+ None => {
+ let mut stage = (Vec::with_capacity(n), Vec::with_capacity(n));
+
+ for _ in 0..n {
+ let i = self.rng.rand_range(0u64..(self.data.0.len() as u64)) as usize;
+
+ stage.0.push(self.data.0[i]);
+ stage.1.push(self.data.1[i]);
+ }
+
+ self.stage = Some(stage);
+ }
+ Some(ref mut stage) => {
+ for i in 0..n {
+ let j = self.rng.rand_range(0u64..(self.data.0.len() as u64)) as usize;
+
+ stage.0[i] = self.data.0[j];
+ stage.1[i] = self.data.1[j];
+ }
+ }
+ }
+
+ if let Some((ref x, ref y)) = self.stage {
+ Data(x, y)
+ } else {
+ unreachable!();
+ }
+ }
+}
diff --git a/src/stats/float.rs b/src/stats/float.rs
new file mode 100755
index 0000000..b7748dd
--- /dev/null
+++ b/src/stats/float.rs
@@ -0,0 +1,15 @@
+//! Float trait
+
+use cast::From;
+use num_traits::float;
+
+/// This is an extension of `num_traits::float::Float` that adds safe
+/// casting and Sync + Send. Once `num_traits` has these features this
+/// can be removed.
+pub trait Float:
+ float::Float + From<usize, Output = Self> + From<f32, Output = Self> + Sync + Send
+{
+}
+
+impl Float for f32 {}
+impl Float for f64 {}
diff --git a/src/stats/mod.rs b/src/stats/mod.rs
new file mode 100755
index 0000000..4f926de
--- /dev/null
+++ b/src/stats/mod.rs
@@ -0,0 +1,112 @@
+//! [Criterion]'s statistics library.
+//!
+//! [Criterion]: https://github.com/bheisler/criterion.rs
+//!
+//! **WARNING** This library is criterion's implementation detail and there no plans to stabilize
+//! it. In other words, the API may break at any time without notice.
+
+#[cfg(test)]
+mod test;
+
+pub mod bivariate;
+pub mod tuple;
+pub mod univariate;
+
+mod float;
+mod rand_util;
+
+use std::mem;
+use std::ops::Deref;
+
+use crate::stats::float::Float;
+use crate::stats::univariate::Sample;
+
+/// The bootstrap distribution of some parameter
+#[derive(Clone)]
+pub struct Distribution<A>(Box<[A]>);
+
+impl<A> Distribution<A>
+where
+ A: Float,
+{
+ /// Create a distribution from the given values
+ pub fn from(values: Box<[A]>) -> Distribution<A> {
+ Distribution(values)
+ }
+
+ /// Computes the confidence interval of the population parameter using percentiles
+ ///
+ /// # Panics
+ ///
+ /// Panics if the `confidence_level` is not in the `(0, 1)` range.
+ pub fn confidence_interval(&self, confidence_level: A) -> (A, A)
+ where
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+ {
+ let _0 = A::cast(0);
+ let _1 = A::cast(1);
+ let _50 = A::cast(50);
+
+ assert!(confidence_level > _0 && confidence_level < _1);
+
+ let percentiles = self.percentiles();
+
+ // FIXME(privacy) this should use the `at_unchecked()` method
+ (
+ percentiles.at(_50 * (_1 - confidence_level)),
+ percentiles.at(_50 * (_1 + confidence_level)),
+ )
+ }
+
+ /// Computes the "likelihood" of seeing the value `t` or "more extreme" values in the
+ /// distribution.
+ pub fn p_value(&self, t: A, tails: &Tails) -> A {
+ use std::cmp;
+
+ let n = self.0.len();
+ let hits = self.0.iter().filter(|&&x| x < t).count();
+
+ let tails = A::cast(match *tails {
+ Tails::One => 1,
+ Tails::Two => 2,
+ });
+
+ A::cast(cmp::min(hits, n - hits)) / A::cast(n) * tails
+ }
+}
+
+impl<A> Deref for Distribution<A> {
+ type Target = Sample<A>;
+
+ fn deref(&self) -> &Sample<A> {
+ let slice: &[_] = &self.0;
+
+ unsafe { mem::transmute(slice) }
+ }
+}
+
+/// Number of tails for significance testing
+pub enum Tails {
+ /// One tailed test
+ One,
+ /// Two tailed test
+ Two,
+}
+
+fn dot<A>(xs: &[A], ys: &[A]) -> A
+where
+ A: Float,
+{
+ xs.iter()
+ .zip(ys)
+ .fold(A::cast(0), |acc, (&x, &y)| acc + x * y)
+}
+
+fn sum<A>(xs: &[A]) -> A
+where
+ A: Float,
+{
+ use std::ops::Add;
+
+ xs.iter().cloned().fold(A::cast(0), Add::add)
+}
diff --git a/src/stats/rand_util.rs b/src/stats/rand_util.rs
new file mode 100755
index 0000000..ed374cf
--- /dev/null
+++ b/src/stats/rand_util.rs
@@ -0,0 +1,21 @@
+use oorandom::Rand64;
+use std::cell::RefCell;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+pub type Rng = Rand64;
+
+thread_local! {
+ static SEED_RAND: RefCell<Rand64> = RefCell::new(Rand64::new(
+ SystemTime::now().duration_since(UNIX_EPOCH)
+ .expect("Time went backwards")
+ .as_millis()
+ ));
+}
+
+pub fn new_rng() -> Rng {
+ SEED_RAND.with(|r| {
+ let mut r = r.borrow_mut();
+ let seed = ((r.rand_u64() as u128) << 64) | (r.rand_u64() as u128);
+ Rand64::new(seed)
+ })
+}
diff --git a/src/stats/test.rs b/src/stats/test.rs
new file mode 100755
index 0000000..9e13f30
--- /dev/null
+++ b/src/stats/test.rs
@@ -0,0 +1,16 @@
+use rand::distributions::{Distribution, Standard};
+use rand::prelude::*;
+use rand::rngs::StdRng;
+
+pub fn vec<T>(size: usize, start: usize) -> Option<Vec<T>>
+where
+ Standard: Distribution<T>,
+{
+ if size > start + 2 {
+ let mut rng = StdRng::from_entropy();
+
+ Some((0..size).map(|_| rng.gen()).collect())
+ } else {
+ None
+ }
+}
diff --git a/src/stats/tuple.rs b/src/stats/tuple.rs
new file mode 100755
index 0000000..1c07515
--- /dev/null
+++ b/src/stats/tuple.rs
@@ -0,0 +1,253 @@
+//! Helper traits for tupling/untupling
+
+use crate::stats::Distribution;
+
+/// Any tuple: `(A, B, ..)`
+pub trait Tuple: Sized {
+ /// A tuple of distributions associated with this tuple
+ type Distributions: TupledDistributions<Item = Self>;
+
+ /// A tuple of vectors associated with this tuple
+ type Builder: TupledDistributionsBuilder<Item = Self>;
+}
+
+/// A tuple of distributions: `(Distribution<A>, Distribution<B>, ..)`
+pub trait TupledDistributions: Sized {
+ /// A tuple that can be pushed/inserted into the tupled distributions
+ type Item: Tuple<Distributions = Self>;
+}
+
+/// A tuple of vecs used to build distributions.
+pub trait TupledDistributionsBuilder: Sized {
+ /// A tuple that can be pushed/inserted into the tupled distributions
+ type Item: Tuple<Builder = Self>;
+
+ /// Creates a new tuple of vecs
+ fn new(size: usize) -> Self;
+
+ /// Push one element into each of the vecs
+ fn push(&mut self, tuple: Self::Item);
+
+ /// Append one tuple of vecs to this one, leaving the vecs in the other tuple empty
+ fn extend(&mut self, other: &mut Self);
+
+ /// Convert the tuple of vectors into a tuple of distributions
+ fn complete(self) -> <Self::Item as Tuple>::Distributions;
+}
+
+impl<A> Tuple for (A,)
+where
+ A: Copy,
+{
+ type Distributions = (Distribution<A>,);
+ type Builder = (Vec<A>,);
+}
+
+impl<A> TupledDistributions for (Distribution<A>,)
+where
+ A: Copy,
+{
+ type Item = (A,);
+}
+impl<A> TupledDistributionsBuilder for (Vec<A>,)
+where
+ A: Copy,
+{
+ type Item = (A,);
+
+ fn new(size: usize) -> (Vec<A>,) {
+ (Vec::with_capacity(size),)
+ }
+
+ fn push(&mut self, tuple: (A,)) {
+ (self.0).push(tuple.0);
+ }
+
+ fn extend(&mut self, other: &mut (Vec<A>,)) {
+ (self.0).append(&mut other.0);
+ }
+
+ fn complete(self) -> (Distribution<A>,) {
+ (Distribution(self.0.into_boxed_slice()),)
+ }
+}
+
+impl<A, B> Tuple for (A, B)
+where
+ A: Copy,
+ B: Copy,
+{
+ type Distributions = (Distribution<A>, Distribution<B>);
+ type Builder = (Vec<A>, Vec<B>);
+}
+
+impl<A, B> TupledDistributions for (Distribution<A>, Distribution<B>)
+where
+ A: Copy,
+ B: Copy,
+{
+ type Item = (A, B);
+}
+impl<A, B> TupledDistributionsBuilder for (Vec<A>, Vec<B>)
+where
+ A: Copy,
+ B: Copy,
+{
+ type Item = (A, B);
+
+ fn new(size: usize) -> (Vec<A>, Vec<B>) {
+ (Vec::with_capacity(size), Vec::with_capacity(size))
+ }
+
+ fn push(&mut self, tuple: (A, B)) {
+ (self.0).push(tuple.0);
+ (self.1).push(tuple.1);
+ }
+
+ fn extend(&mut self, other: &mut (Vec<A>, Vec<B>)) {
+ (self.0).append(&mut other.0);
+ (self.1).append(&mut other.1);
+ }
+
+ fn complete(self) -> (Distribution<A>, Distribution<B>) {
+ (
+ Distribution(self.0.into_boxed_slice()),
+ Distribution(self.1.into_boxed_slice()),
+ )
+ }
+}
+
+impl<A, B, C> Tuple for (A, B, C)
+where
+ A: Copy,
+ B: Copy,
+ C: Copy,
+{
+ type Distributions = (Distribution<A>, Distribution<B>, Distribution<C>);
+ type Builder = (Vec<A>, Vec<B>, Vec<C>);
+}
+
+impl<A, B, C> TupledDistributions for (Distribution<A>, Distribution<B>, Distribution<C>)
+where
+ A: Copy,
+ B: Copy,
+ C: Copy,
+{
+ type Item = (A, B, C);
+}
+impl<A, B, C> TupledDistributionsBuilder for (Vec<A>, Vec<B>, Vec<C>)
+where
+ A: Copy,
+ B: Copy,
+ C: Copy,
+{
+ type Item = (A, B, C);
+
+ fn new(size: usize) -> (Vec<A>, Vec<B>, Vec<C>) {
+ (
+ Vec::with_capacity(size),
+ Vec::with_capacity(size),
+ Vec::with_capacity(size),
+ )
+ }
+
+ fn push(&mut self, tuple: (A, B, C)) {
+ (self.0).push(tuple.0);
+ (self.1).push(tuple.1);
+ (self.2).push(tuple.2);
+ }
+
+ fn extend(&mut self, other: &mut (Vec<A>, Vec<B>, Vec<C>)) {
+ (self.0).append(&mut other.0);
+ (self.1).append(&mut other.1);
+ (self.2).append(&mut other.2);
+ }
+
+ fn complete(self) -> (Distribution<A>, Distribution<B>, Distribution<C>) {
+ (
+ Distribution(self.0.into_boxed_slice()),
+ Distribution(self.1.into_boxed_slice()),
+ Distribution(self.2.into_boxed_slice()),
+ )
+ }
+}
+
+impl<A, B, C, D> Tuple for (A, B, C, D)
+where
+ A: Copy,
+ B: Copy,
+ C: Copy,
+ D: Copy,
+{
+ type Distributions = (
+ Distribution<A>,
+ Distribution<B>,
+ Distribution<C>,
+ Distribution<D>,
+ );
+ type Builder = (Vec<A>, Vec<B>, Vec<C>, Vec<D>);
+}
+
+impl<A, B, C, D> TupledDistributions
+ for (
+ Distribution<A>,
+ Distribution<B>,
+ Distribution<C>,
+ Distribution<D>,
+ )
+where
+ A: Copy,
+ B: Copy,
+ C: Copy,
+ D: Copy,
+{
+ type Item = (A, B, C, D);
+}
+impl<A, B, C, D> TupledDistributionsBuilder for (Vec<A>, Vec<B>, Vec<C>, Vec<D>)
+where
+ A: Copy,
+ B: Copy,
+ C: Copy,
+ D: Copy,
+{
+ type Item = (A, B, C, D);
+
+ fn new(size: usize) -> (Vec<A>, Vec<B>, Vec<C>, Vec<D>) {
+ (
+ Vec::with_capacity(size),
+ Vec::with_capacity(size),
+ Vec::with_capacity(size),
+ Vec::with_capacity(size),
+ )
+ }
+
+ fn push(&mut self, tuple: (A, B, C, D)) {
+ (self.0).push(tuple.0);
+ (self.1).push(tuple.1);
+ (self.2).push(tuple.2);
+ (self.3).push(tuple.3);
+ }
+
+ fn extend(&mut self, other: &mut (Vec<A>, Vec<B>, Vec<C>, Vec<D>)) {
+ (self.0).append(&mut other.0);
+ (self.1).append(&mut other.1);
+ (self.2).append(&mut other.2);
+ (self.3).append(&mut other.3);
+ }
+
+ fn complete(
+ self,
+ ) -> (
+ Distribution<A>,
+ Distribution<B>,
+ Distribution<C>,
+ Distribution<D>,
+ ) {
+ (
+ Distribution(self.0.into_boxed_slice()),
+ Distribution(self.1.into_boxed_slice()),
+ Distribution(self.2.into_boxed_slice()),
+ Distribution(self.3.into_boxed_slice()),
+ )
+ }
+}
diff --git a/src/stats/univariate/bootstrap.rs b/src/stats/univariate/bootstrap.rs
new file mode 100755
index 0000000..dbb52f5
--- /dev/null
+++ b/src/stats/univariate/bootstrap.rs
@@ -0,0 +1,161 @@
+#[cfg(test)]
+macro_rules! test {
+ ($ty:ident) => {
+ mod $ty {
+ use approx::relative_eq;
+ use quickcheck::quickcheck;
+ use quickcheck::TestResult;
+
+ use crate::stats::univariate::{Sample, mixed, self};
+
+ quickcheck!{
+ fn mean(size: usize, start: usize, nresamples: usize) -> TestResult {
+ if let Some(v) = crate::stats::test::vec::<$ty>(size, start) {
+ let sample = Sample::new(&v[start..]);
+
+ let means = if nresamples > 0 {
+ sample.bootstrap(nresamples, |s| (s.mean(),)).0
+ } else {
+ return TestResult::discard();
+ };
+
+ let min = sample.min();
+ let max = sample.max();
+
+ TestResult::from_bool(
+ // Computed the correct number of resamples
+ means.len() == nresamples &&
+ // No uninitialized values
+ means.iter().all(|&x| {
+ (x > min || relative_eq!(x, min)) &&
+ (x < max || relative_eq!(x, max))
+ })
+ )
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+
+ quickcheck!{
+ fn mean_median(size: usize, start: usize, nresamples: usize) -> TestResult {
+ if let Some(v) = crate::stats::test::vec::<$ty>(size, start) {
+ let sample = Sample::new(&v[start..]);
+
+ let (means, medians) = if nresamples > 0 {
+ sample.bootstrap(nresamples, |s| (s.mean(), s.median()))
+ } else {
+ return TestResult::discard();
+ };
+
+ let min = sample.min();
+ let max = sample.max();
+
+ TestResult::from_bool(
+ // Computed the correct number of resamples
+ means.len() == nresamples &&
+ medians.len() == nresamples &&
+ // No uninitialized values
+ means.iter().all(|&x| {
+ (x > min || relative_eq!(x, min)) &&
+ (x < max || relative_eq!(x, max))
+ }) &&
+ medians.iter().all(|&x| {
+ (x > min || relative_eq!(x, min)) &&
+ (x < max || relative_eq!(x, max))
+ })
+ )
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+
+ quickcheck!{
+ fn mixed_two_sample(
+ a_size: usize, a_start: usize,
+ b_size: usize, b_start: usize,
+ nresamples: usize
+ ) -> TestResult {
+ if let (Some(a), Some(b)) =
+ (crate::stats::test::vec::<$ty>(a_size, a_start), crate::stats::test::vec::<$ty>(b_size, b_start))
+ {
+ let a = Sample::new(&a);
+ let b = Sample::new(&b);
+
+ let distribution = if nresamples > 0 {
+ mixed::bootstrap(a, b, nresamples, |a, b| (a.mean() - b.mean(),)).0
+ } else {
+ return TestResult::discard();
+ };
+
+ let min = <$ty>::min(a.min() - b.max(), b.min() - a.max());
+ let max = <$ty>::max(a.max() - b.min(), b.max() - a.min());
+
+ TestResult::from_bool(
+ // Computed the correct number of resamples
+ distribution.len() == nresamples &&
+ // No uninitialized values
+ distribution.iter().all(|&x| {
+ (x > min || relative_eq!(x, min)) &&
+ (x < max || relative_eq!(x, max))
+ })
+ )
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+
+ quickcheck!{
+ fn two_sample(
+ a_size: usize, a_start: usize,
+ b_size: usize, b_start: usize,
+ nresamples: usize
+ ) -> TestResult {
+ if let (Some(a), Some(b)) =
+ (crate::stats::test::vec::<$ty>(a_size, a_start), crate::stats::test::vec::<$ty>(b_size, b_start))
+ {
+ let a = Sample::new(&a[a_start..]);
+ let b = Sample::new(&b[b_start..]);
+
+ let distribution = if nresamples > 0 {
+ univariate::bootstrap(a, b, nresamples, |a, b| (a.mean() - b.mean(),)).0
+ } else {
+ return TestResult::discard();
+ };
+
+ let min = <$ty>::min(a.min() - b.max(), b.min() - a.max());
+ let max = <$ty>::max(a.max() - b.min(), b.max() - a.min());
+
+ // Computed the correct number of resamples
+ let pass = distribution.len() == nresamples &&
+ // No uninitialized values
+ distribution.iter().all(|&x| {
+ (x > min || relative_eq!(x, min)) &&
+ (x < max || relative_eq!(x, max))
+ });
+
+ if !pass {
+ println!("A: {:?} (len={})", a.as_ref(), a.len());
+ println!("B: {:?} (len={})", b.as_ref(), b.len());
+ println!("Dist: {:?} (len={})", distribution.as_ref(), distribution.len());
+ println!("Min: {}, Max: {}, nresamples: {}",
+ min, max, nresamples);
+ }
+
+ TestResult::from_bool(pass)
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ test!(f32);
+ test!(f64);
+}
diff --git a/src/stats/univariate/kde/kernel.rs b/src/stats/univariate/kde/kernel.rs
new file mode 100755
index 0000000..b4204f5
--- /dev/null
+++ b/src/stats/univariate/kde/kernel.rs
@@ -0,0 +1,82 @@
+//! Kernels
+
+use crate::stats::float::Float;
+
+/// Kernel function
+pub trait Kernel<A>: Copy + Sync
+where
+ A: Float,
+{
+ /// Apply the kernel function to the given x-value.
+ fn evaluate(&self, x: A) -> A;
+}
+
+/// Gaussian kernel
+#[derive(Clone, Copy)]
+pub struct Gaussian;
+
+impl<A> Kernel<A> for Gaussian
+where
+ A: Float,
+{
+ fn evaluate(&self, x: A) -> A {
+ use std::f32::consts::PI;
+
+ (x.powi(2).exp() * A::cast(2. * PI)).sqrt().recip()
+ }
+}
+
+#[cfg(test)]
+macro_rules! test {
+ ($ty:ident) => {
+ mod $ty {
+ mod gaussian {
+ use approx::relative_eq;
+ use quickcheck::quickcheck;
+ use quickcheck::TestResult;
+
+ use crate::stats::univariate::kde::kernel::{Gaussian, Kernel};
+
+ quickcheck! {
+ fn symmetric(x: $ty) -> bool {
+ relative_eq!(Gaussian.evaluate(-x), Gaussian.evaluate(x))
+ }
+ }
+
+ // Any [a b] integral should be in the range [0 1]
+ quickcheck! {
+ fn integral(a: $ty, b: $ty) -> TestResult {
+ const DX: $ty = 1e-3;
+
+ if a > b {
+ TestResult::discard()
+ } else {
+ let mut acc = 0.;
+ let mut x = a;
+ let mut y = Gaussian.evaluate(a);
+
+ while x < b {
+ acc += DX * y / 2.;
+
+ x += DX;
+ y = Gaussian.evaluate(x);
+
+ acc += DX * y / 2.;
+ }
+
+ TestResult::from_bool(
+ (acc > 0. || relative_eq!(acc, 0.)) &&
+ (acc < 1. || relative_eq!(acc, 1.)))
+ }
+ }
+ }
+ }
+ }
+ };
+}
+
+#[cfg(test)]
+mod test {
+ test!(f32);
+ test!(f64);
+}
diff --git a/src/stats/univariate/kde/mod.rs b/src/stats/univariate/kde/mod.rs
new file mode 100755
index 0000000..efc27cd
--- /dev/null
+++ b/src/stats/univariate/kde/mod.rs
@@ -0,0 +1,140 @@
+//! Kernel density estimation
+
+pub mod kernel;
+
+use self::kernel::Kernel;
+use crate::stats::float::Float;
+use crate::stats::univariate::Sample;
+use rayon::prelude::*;
+
+/// Univariate kernel density estimator
+pub struct Kde<'a, A, K>
+where
+ A: Float,
+ K: Kernel<A>,
+{
+ bandwidth: A,
+ kernel: K,
+ sample: &'a Sample<A>,
+}
+
+impl<'a, A, K> Kde<'a, A, K>
+where
+ A: 'a + Float,
+ K: Kernel<A>,
+{
+ /// Creates a new kernel density estimator from the `sample`, using a kernel and estimating
+ /// the bandwidth using the method `bw`
+ pub fn new(sample: &'a Sample<A>, kernel: K, bw: Bandwidth) -> Kde<'a, A, K> {
+ Kde {
+ bandwidth: bw.estimate(sample),
+ kernel,
+ sample,
+ }
+ }
+
+ /// Returns the bandwidth used by the estimator
+ pub fn bandwidth(&self) -> A {
+ self.bandwidth
+ }
+
+ /// Maps the KDE over `xs`
+ ///
+ /// - Multihreaded
+ pub fn map(&self, xs: &[A]) -> Box<[A]> {
+ xs.par_iter()
+ .map(|&x| self.estimate(x))
+ .collect::<Vec<_>>()
+ .into_boxed_slice()
+ }
+
+ /// Estimates the probability density of `x`
+ pub fn estimate(&self, x: A) -> A {
+ let _0 = A::cast(0);
+ let slice = self.sample;
+ let h = self.bandwidth;
+ let n = A::cast(slice.len());
+
+ let sum = slice
+ .iter()
+ .fold(_0, |acc, &x_i| acc + self.kernel.evaluate((x - x_i) / h));
+
+ sum / (h * n)
+ }
+}
+
+/// Method to estimate the bandwidth
+pub enum Bandwidth {
+ /// Use Silverman's rule of thumb to estimate the bandwidth from the sample
+ Silverman,
+}
+
+impl Bandwidth {
+ fn estimate<A: Float>(self, sample: &Sample<A>) -> A {
+ match self {
+ Bandwidth::Silverman => {
+ let factor = A::cast(4. / 3.);
+ let exponent = A::cast(1. / 5.);
+ let n = A::cast(sample.len());
+ let sigma = sample.std_dev(None);
+
+ sigma * (factor / n).powf(exponent)
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+macro_rules! test {
+ ($ty:ident) => {
+ mod $ty {
+ use approx::relative_eq;
+ use quickcheck::quickcheck;
+ use quickcheck::TestResult;
+
+ use crate::stats::univariate::kde::kernel::Gaussian;
+ use crate::stats::univariate::kde::{Bandwidth, Kde};
+ use crate::stats::univariate::Sample;
+
+ // The [-inf inf] integral of the estimated PDF should be one
+ quickcheck! {
+ fn integral(size: usize, start: usize) -> TestResult {
+ const DX: $ty = 1e-3;
+
+ if let Some(v) = crate::stats::test::vec::<$ty>(size, start) {
+ let slice = &v[start..];
+ let data = Sample::new(slice);
+ let kde = Kde::new(data, Gaussian, Bandwidth::Silverman);
+ let h = kde.bandwidth();
+ // NB Obviously a [-inf inf] integral is not feasible, but this range works
+ // quite well
+ let (a, b) = (data.min() - 5. * h, data.max() + 5. * h);
+
+ let mut acc = 0.;
+ let mut x = a;
+ let mut y = kde.estimate(a);
+
+ while x < b {
+ acc += DX * y / 2.;
+
+ x += DX;
+ y = kde.estimate(x);
+
+ acc += DX * y / 2.;
+ }
+
+ TestResult::from_bool(relative_eq!(acc, 1., epsilon = 2e-5))
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+ }
+ };
+}
+
+#[cfg(test)]
+mod test {
+ test!(f32);
+ test!(f64);
+}
diff --git a/src/stats/univariate/mixed.rs b/src/stats/univariate/mixed.rs
new file mode 100755
index 0000000..5c0a59f
--- /dev/null
+++ b/src/stats/univariate/mixed.rs
@@ -0,0 +1,57 @@
+//! Mixed bootstrap
+
+use crate::stats::float::Float;
+use crate::stats::tuple::{Tuple, TupledDistributionsBuilder};
+use crate::stats::univariate::Resamples;
+use crate::stats::univariate::Sample;
+use rayon::prelude::*;
+
+/// Performs a *mixed* two-sample bootstrap
+pub fn bootstrap<A, T, S>(
+ a: &Sample<A>,
+ b: &Sample<A>,
+ nresamples: usize,
+ statistic: S,
+) -> T::Distributions
+where
+ A: Float,
+ S: Fn(&Sample<A>, &Sample<A>) -> T + Sync,
+ T: Tuple + Send,
+ T::Distributions: Send,
+ T::Builder: Send,
+{
+ let n_a = a.len();
+ let n_b = b.len();
+ let mut c = Vec::with_capacity(n_a + n_b);
+ c.extend_from_slice(a);
+ c.extend_from_slice(b);
+ let c = Sample::new(&c);
+
+ (0..nresamples)
+ .into_par_iter()
+ .map_init(
+ || Resamples::new(c),
+ |resamples, _| {
+ let resample = resamples.next();
+ let a: &Sample<A> = Sample::new(&resample[..n_a]);
+ let b: &Sample<A> = Sample::new(&resample[n_a..]);
+
+ statistic(a, b)
+ },
+ )
+ .fold(
+ || T::Builder::new(0),
+ |mut sub_distributions, sample| {
+ sub_distributions.push(sample);
+ sub_distributions
+ },
+ )
+ .reduce(
+ || T::Builder::new(0),
+ |mut a, mut b| {
+ a.extend(&mut b);
+ a
+ },
+ )
+ .complete()
+}
diff --git a/src/stats/univariate/mod.rs b/src/stats/univariate/mod.rs
new file mode 100755
index 0000000..8dfb5f8
--- /dev/null
+++ b/src/stats/univariate/mod.rs
@@ -0,0 +1,72 @@
+//! Univariate analysis
+
+mod bootstrap;
+mod percentiles;
+mod resamples;
+mod sample;
+
+pub mod kde;
+pub mod mixed;
+pub mod outliers;
+
+use crate::stats::float::Float;
+use crate::stats::tuple::{Tuple, TupledDistributionsBuilder};
+use rayon::prelude::*;
+use std::cmp;
+
+use self::resamples::Resamples;
+
+pub use self::percentiles::Percentiles;
+pub use self::sample::Sample;
+
+/// Performs a two-sample bootstrap
+///
+/// - Multithreaded
+/// - Time: `O(nresamples)`
+/// - Memory: `O(nresamples)`
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_lossless))]
+pub fn bootstrap<A, B, T, S>(
+ a: &Sample<A>,
+ b: &Sample<B>,
+ nresamples: usize,
+ statistic: S,
+) -> T::Distributions
+where
+ A: Float,
+ B: Float,
+ S: Fn(&Sample<A>, &Sample<B>) -> T + Sync,
+ T: Tuple + Send,
+ T::Distributions: Send,
+ T::Builder: Send,
+{
+ let nresamples_sqrt = (nresamples as f64).sqrt().ceil() as usize;
+ let per_chunk = (nresamples + nresamples_sqrt - 1) / nresamples_sqrt;
+
+ (0..nresamples_sqrt)
+ .into_par_iter()
+ .map_init(
+ || (Resamples::new(a), Resamples::new(b)),
+ |(a_resamples, b_resamples), i| {
+ let start = i * per_chunk;
+ let end = cmp::min((i + 1) * per_chunk, nresamples);
+ let a_resample = a_resamples.next();
+
+ let mut sub_distributions: T::Builder =
+ TupledDistributionsBuilder::new(end - start);
+
+ for _ in start..end {
+ let b_resample = b_resamples.next();
+ sub_distributions.push(statistic(a_resample, b_resample));
+ }
+ sub_distributions
+ },
+ )
+ .reduce(
+ || T::Builder::new(0),
+ |mut a, mut b| {
+ a.extend(&mut b);
+ a
+ },
+ )
+ .complete()
+}
diff --git a/src/stats/univariate/outliers/mod.rs b/src/stats/univariate/outliers/mod.rs
new file mode 100755
index 0000000..b8ed7c7
--- /dev/null
+++ b/src/stats/univariate/outliers/mod.rs
@@ -0,0 +1,7 @@
+//! Classification of outliers
+//!
+//! WARNING: There's no formal/mathematical definition of what an outlier actually is. Therefore,
+//! all outlier classifiers are *subjective*, however some classifiers that have become *de facto*
+//! standard are provided here.
+
+pub mod tukey;
diff --git a/src/stats/univariate/outliers/tukey.rs b/src/stats/univariate/outliers/tukey.rs
new file mode 100755
index 0000000..bfd08f1
--- /dev/null
+++ b/src/stats/univariate/outliers/tukey.rs
@@ -0,0 +1,291 @@
+//! Tukey's method
+//!
+//! The original method uses two "fences" to classify the data. All the observations "inside" the
+//! fences are considered "normal", and the rest are considered outliers.
+//!
+//! The fences are computed from the quartiles of the sample, according to the following formula:
+//!
+//! ``` ignore
+//! // q1, q3 are the first and third quartiles
+//! let iqr = q3 - q1; // The interquartile range
+//! let (f1, f2) = (q1 - 1.5 * iqr, q3 + 1.5 * iqr); // the "fences"
+//!
+//! let is_outlier = |x| if x > f1 && x < f2 { true } else { false };
+//! ```
+//!
+//! The classifier provided here adds two extra outer fences:
+//!
+//! ``` ignore
+//! let (f3, f4) = (q1 - 3 * iqr, q3 + 3 * iqr); // the outer "fences"
+//! ```
+//!
+//! The extra fences add a sense of "severity" to the classification. Data points outside of the
+//! outer fences are considered "severe" outliers, whereas points outside the inner fences are just
+//! "mild" outliers, and, as the original method, everything inside the inner fences is considered
+//! "normal" data.
+//!
+//! Some ASCII art for the visually oriented people:
+//!
+//! ``` ignore
+//! LOW-ish NORMAL-ish HIGH-ish
+//! x | + | o o o o o o o | + | x
+//! f3 f1 f2 f4
+//!
+//! Legend:
+//! o: "normal" data (not an outlier)
+//! +: "mild" outlier
+//! x: "severe" outlier
+//! ```
+
+use std::iter::IntoIterator;
+use std::ops::{Deref, Index};
+use std::slice;
+
+use crate::stats::float::Float;
+use crate::stats::univariate::Sample;
+
+use self::Label::*;
+
+/// A classified/labeled sample.
+///
+/// The labeled data can be accessed using the indexing operator. The order of the data points is
+/// retained.
+///
+/// NOTE: Due to limitations in the indexing traits, only the label is returned. Once the
+/// `IndexGet` trait lands in stdlib, the indexing operation will return a `(data_point, label)`
+/// pair.
+#[derive(Clone, Copy)]
+pub struct LabeledSample<'a, A>
+where
+ A: Float,
+{
+ fences: (A, A, A, A),
+ sample: &'a Sample<A>,
+}
+
+impl<'a, A> LabeledSample<'a, A>
+where
+ A: Float,
+{
+ /// Returns the number of data points per label
+ ///
+ /// - Time: `O(length)`
+ #[cfg_attr(feature = "cargo-clippy", allow(clippy::similar_names))]
+ pub fn count(&self) -> (usize, usize, usize, usize, usize) {
+ let (mut los, mut lom, mut noa, mut him, mut his) = (0, 0, 0, 0, 0);
+
+ for (_, label) in self {
+ match label {
+ LowSevere => {
+ los += 1;
+ }
+ LowMild => {
+ lom += 1;
+ }
+ NotAnOutlier => {
+ noa += 1;
+ }
+ HighMild => {
+ him += 1;
+ }
+ HighSevere => {
+ his += 1;
+ }
+ }
+ }
+
+ (los, lom, noa, him, his)
+ }
+
+ /// Returns the fences used to classify the outliers
+ pub fn fences(&self) -> (A, A, A, A) {
+ self.fences
+ }
+
+ /// Returns an iterator over the labeled data
+ pub fn iter(&self) -> Iter<'a, A> {
+ Iter {
+ fences: self.fences,
+ iter: self.sample.iter(),
+ }
+ }
+}
+
+impl<'a, A> Deref for LabeledSample<'a, A>
+where
+ A: Float,
+{
+ type Target = Sample<A>;
+
+ fn deref(&self) -> &Sample<A> {
+ self.sample
+ }
+}
+
+// FIXME Use the `IndexGet` trait
+impl<'a, A> Index<usize> for LabeledSample<'a, A>
+where
+ A: Float,
+{
+ type Output = Label;
+
+ #[cfg_attr(feature = "cargo-clippy", allow(clippy::similar_names))]
+ fn index(&self, i: usize) -> &Label {
+ static LOW_SEVERE: Label = LowSevere;
+ static LOW_MILD: Label = LowMild;
+ static HIGH_MILD: Label = HighMild;
+ static HIGH_SEVERE: Label = HighSevere;
+ static NOT_AN_OUTLIER: Label = NotAnOutlier;
+
+ let x = self.sample[i];
+ let (lost, lomt, himt, hist) = self.fences;
+
+ if x < lost {
+ &LOW_SEVERE
+ } else if x > hist {
+ &HIGH_SEVERE
+ } else if x < lomt {
+ &LOW_MILD
+ } else if x > himt {
+ &HIGH_MILD
+ } else {
+ &NOT_AN_OUTLIER
+ }
+ }
+}
+
+impl<'a, 'b, A> IntoIterator for &'b LabeledSample<'a, A>
+where
+ A: Float,
+{
+ type Item = (A, Label);
+ type IntoIter = Iter<'a, A>;
+
+ fn into_iter(self) -> Iter<'a, A> {
+ self.iter()
+ }
+}
+
+/// Iterator over the labeled data
+pub struct Iter<'a, A>
+where
+ A: Float,
+{
+ fences: (A, A, A, A),
+ iter: slice::Iter<'a, A>,
+}
+
+impl<'a, A> Iterator for Iter<'a, A>
+where
+ A: Float,
+{
+ type Item = (A, Label);
+
+ #[cfg_attr(feature = "cargo-clippy", allow(clippy::similar_names))]
+ fn next(&mut self) -> Option<(A, Label)> {
+ self.iter.next().map(|&x| {
+ let (lost, lomt, himt, hist) = self.fences;
+
+ let label = if x < lost {
+ LowSevere
+ } else if x > hist {
+ HighSevere
+ } else if x < lomt {
+ LowMild
+ } else if x > himt {
+ HighMild
+ } else {
+ NotAnOutlier
+ };
+
+ (x, label)
+ })
+ }
+
+ fn size_hint(&self) -> (usize, Option<usize>) {
+ self.iter.size_hint()
+ }
+}
+
+/// Labels used to classify outliers
+pub enum Label {
+ /// A "mild" outlier in the "high" spectrum
+ HighMild,
+ /// A "severe" outlier in the "high" spectrum
+ HighSevere,
+ /// A "mild" outlier in the "low" spectrum
+ LowMild,
+ /// A "severe" outlier in the "low" spectrum
+ LowSevere,
+ /// A normal data point
+ NotAnOutlier,
+}
+
+impl Label {
+ /// Checks if the data point has an "unusually" high value
+ pub fn is_high(&self) -> bool {
+ match *self {
+ HighMild | HighSevere => true,
+ _ => false,
+ }
+ }
+
+ /// Checks if the data point is labeled as a "mild" outlier
+ pub fn is_mild(&self) -> bool {
+ match *self {
+ HighMild | LowMild => true,
+ _ => false,
+ }
+ }
+
+ /// Checks if the data point has an "unusually" low value
+ pub fn is_low(&self) -> bool {
+ match *self {
+ LowMild | LowSevere => true,
+ _ => false,
+ }
+ }
+
+ /// Checks if the data point is labeled as an outlier
+ pub fn is_outlier(&self) -> bool {
+ match *self {
+ NotAnOutlier => false,
+ _ => true,
+ }
+ }
+
+ /// Checks if the data point is labeled as a "severe" outlier
+ pub fn is_severe(&self) -> bool {
+ match *self {
+ HighSevere | LowSevere => true,
+ _ => false,
+ }
+ }
+}
+
+/// Classifies the sample, and returns a labeled sample.
+///
+/// - Time: `O(N log N) where N = length`
+pub fn classify<A>(sample: &Sample<A>) -> LabeledSample<'_, A>
+where
+ A: Float,
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+{
+ let (q1, _, q3) = sample.percentiles().quartiles();
+ let iqr = q3 - q1;
+
+ // Mild
+ let k_m = A::cast(1.5_f32);
+ // Severe
+ let k_s = A::cast(3);
+
+ LabeledSample {
+ fences: (
+ q1 - k_s * iqr,
+ q1 - k_m * iqr,
+ q3 + k_m * iqr,
+ q3 + k_s * iqr,
+ ),
+ sample,
+ }
+}
diff --git a/src/stats/univariate/percentiles.rs b/src/stats/univariate/percentiles.rs
new file mode 100755
index 0000000..be6bcf3
--- /dev/null
+++ b/src/stats/univariate/percentiles.rs
@@ -0,0 +1,80 @@
+use crate::stats::float::Float;
+use cast::{self, usize};
+
+/// A "view" into the percentiles of a sample
+pub struct Percentiles<A>(Box<[A]>)
+where
+ A: Float;
+
+// TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module
+impl<A> Percentiles<A>
+where
+ A: Float,
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+{
+ /// Returns the percentile at `p`%
+ ///
+ /// Safety:
+ ///
+ /// - Make sure that `p` is in the range `[0, 100]`
+ unsafe fn at_unchecked(&self, p: A) -> A {
+ let _100 = A::cast(100);
+ debug_assert!(p >= A::cast(0) && p <= _100);
+ debug_assert!(self.0.len() > 0);
+ let len = self.0.len() - 1;
+
+ if p == _100 {
+ self.0[len]
+ } else {
+ let rank = (p / _100) * A::cast(len);
+ let integer = rank.floor();
+ let fraction = rank - integer;
+ let n = usize(integer).unwrap();
+ let &floor = self.0.get_unchecked(n);
+ let &ceiling = self.0.get_unchecked(n + 1);
+
+ floor + (ceiling - floor) * fraction
+ }
+ }
+
+ /// Returns the percentile at `p`%
+ ///
+ /// # Panics
+ ///
+ /// Panics if `p` is outside the closed `[0, 100]` range
+ pub fn at(&self, p: A) -> A {
+ let _0 = A::cast(0);
+ let _100 = A::cast(100);
+
+ assert!(p >= _0 && p <= _100);
+ assert!(self.0.len() > 0);
+
+ unsafe { self.at_unchecked(p) }
+ }
+
+ /// Returns the interquartile range
+ pub fn iqr(&self) -> A {
+ unsafe {
+ let q1 = self.at_unchecked(A::cast(25));
+ let q3 = self.at_unchecked(A::cast(75));
+
+ q3 - q1
+ }
+ }
+
+ /// Returns the 50th percentile
+ pub fn median(&self) -> A {
+ unsafe { self.at_unchecked(A::cast(50)) }
+ }
+
+ /// Returns the 25th, 50th and 75th percentiles
+ pub fn quartiles(&self) -> (A, A, A) {
+ unsafe {
+ (
+ self.at_unchecked(A::cast(25)),
+ self.at_unchecked(A::cast(50)),
+ self.at_unchecked(A::cast(75)),
+ )
+ }
+ }
+}
diff --git a/src/stats/univariate/resamples.rs b/src/stats/univariate/resamples.rs
new file mode 100755
index 0000000..831bc7a
--- /dev/null
+++ b/src/stats/univariate/resamples.rs
@@ -0,0 +1,117 @@
+use std::mem;
+
+use crate::stats::float::Float;
+use crate::stats::rand_util::{new_rng, Rng};
+use crate::stats::univariate::Sample;
+
+pub struct Resamples<'a, A>
+where
+ A: Float,
+{
+ rng: Rng,
+ sample: &'a [A],
+ stage: Option<Vec<A>>,
+}
+
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))]
+impl<'a, A> Resamples<'a, A>
+where
+ A: 'a + Float,
+{
+ pub fn new(sample: &'a Sample<A>) -> Resamples<'a, A> {
+ let slice = sample;
+
+ Resamples {
+ rng: new_rng(),
+ sample: slice,
+ stage: None,
+ }
+ }
+
+ pub fn next(&mut self) -> &Sample<A> {
+ let n = self.sample.len();
+ let rng = &mut self.rng;
+
+ match self.stage {
+ None => {
+ let mut stage = Vec::with_capacity(n);
+
+ for _ in 0..n {
+ let idx = rng.rand_range(0u64..(self.sample.len() as u64));
+ stage.push(self.sample[idx as usize])
+ }
+
+ self.stage = Some(stage);
+ }
+ Some(ref mut stage) => {
+ for elem in stage.iter_mut() {
+ let idx = rng.rand_range(0u64..(self.sample.len() as u64));
+ *elem = self.sample[idx as usize]
+ }
+ }
+ }
+
+ if let Some(ref v) = self.stage {
+ unsafe { mem::transmute::<&[_], _>(v) }
+ } else {
+ unreachable!();
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use quickcheck::quickcheck;
+ use quickcheck::TestResult;
+ use std::collections::HashSet;
+
+ use crate::stats::univariate::resamples::Resamples;
+ use crate::stats::univariate::Sample;
+
+ // Check that the resample is a subset of the sample
+ quickcheck! {
+ fn subset(size: usize, nresamples: usize) -> TestResult {
+ if size > 1 {
+ let v: Vec<_> = (0..size).map(|i| i as f32).collect();
+ let sample = Sample::new(&v);
+ let mut resamples = Resamples::new(sample);
+ let sample = v.iter().map(|&x| x as i64).collect::<HashSet<_>>();
+
+ TestResult::from_bool((0..nresamples).all(|_| {
+ let resample = resamples.next()
+
+ .iter()
+ .map(|&x| x as i64)
+ .collect::<HashSet<_>>();
+
+ resample.is_subset(&sample)
+ }))
+ } else {
+ TestResult::discard()
+ }
+ }
+ }
+
+ #[test]
+ fn different_subsets() {
+ let size = 1000;
+ let v: Vec<_> = (0..size).map(|i| i as f32).collect();
+ let sample = Sample::new(&v);
+ let mut resamples = Resamples::new(sample);
+
+ // Hypothetically, we might see one duplicate, but more than one is likely to be a bug.
+ let mut num_duplicated = 0;
+ for _ in 0..1000 {
+ let sample_1 = resamples.next().iter().cloned().collect::<Vec<_>>();
+ let sample_2 = resamples.next().iter().cloned().collect::<Vec<_>>();
+
+ if sample_1 == sample_2 {
+ num_duplicated += 1;
+ }
+ }
+
+ if num_duplicated > 1 {
+ panic!("Found {} duplicate samples", num_duplicated);
+ }
+ }
+}
diff --git a/src/stats/univariate/sample.rs b/src/stats/univariate/sample.rs
new file mode 100755
index 0000000..8f10db7
--- /dev/null
+++ b/src/stats/univariate/sample.rs
@@ -0,0 +1,255 @@
+use std::{mem, ops};
+
+use crate::stats::float::Float;
+use crate::stats::tuple::{Tuple, TupledDistributionsBuilder};
+use crate::stats::univariate::Percentiles;
+use crate::stats::univariate::Resamples;
+use rayon::prelude::*;
+
+/// A collection of data points drawn from a population
+///
+/// Invariants:
+///
+/// - The sample contains at least 2 data points
+/// - The sample contains no `NaN`s
+pub struct Sample<A>([A]);
+
+// TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module
+impl<A> Sample<A>
+where
+ A: Float,
+{
+ /// Creates a new sample from an existing slice
+ ///
+ /// # Panics
+ ///
+ /// Panics if `slice` contains any `NaN` or if `slice` has less than two elements
+ #[cfg_attr(feature = "cargo-clippy", allow(clippy::new_ret_no_self))]
+ pub fn new(slice: &[A]) -> &Sample<A> {
+ assert!(slice.len() > 1 && slice.iter().all(|x| !x.is_nan()));
+
+ unsafe { mem::transmute(slice) }
+ }
+
+ /// Returns the biggest element in the sample
+ ///
+ /// - Time: `O(length)`
+ pub fn max(&self) -> A {
+ let mut elems = self.iter();
+
+ match elems.next() {
+ Some(&head) => elems.fold(head, |a, &b| a.max(b)),
+ // NB `unreachable!` because `Sample` is guaranteed to have at least one data point
+ None => unreachable!(),
+ }
+ }
+
+ /// Returns the arithmetic average of the sample
+ ///
+ /// - Time: `O(length)`
+ pub fn mean(&self) -> A {
+ let n = self.len();
+
+ self.sum() / A::cast(n)
+ }
+
+ /// Returns the median absolute deviation
+ ///
+ /// The `median` can be optionally passed along to speed up (2X) the computation
+ ///
+ /// - Time: `O(length)`
+ /// - Memory: `O(length)`
+ pub fn median_abs_dev(&self, median: Option<A>) -> A
+ where
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+ {
+ let median = median.unwrap_or_else(|| self.percentiles().median());
+
+ // NB Although this operation can be SIMD accelerated, the gain is negligible because the
+ // bottle neck is the sorting operation which is part of the computation of the median
+ let abs_devs = self.iter().map(|&x| (x - median).abs()).collect::<Vec<_>>();
+
+ let abs_devs: &Self = Self::new(&abs_devs);
+
+ abs_devs.percentiles().median() * A::cast(1.4826)
+ }
+
+ /// Returns the median absolute deviation as a percentage of the median
+ ///
+ /// - Time: `O(length)`
+ /// - Memory: `O(length)`
+ pub fn median_abs_dev_pct(&self) -> A
+ where
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+ {
+ let _100 = A::cast(100);
+ let median = self.percentiles().median();
+ let mad = self.median_abs_dev(Some(median));
+
+ (mad / median) * _100
+ }
+
+ /// Returns the smallest element in the sample
+ ///
+ /// - Time: `O(length)`
+ pub fn min(&self) -> A {
+ let mut elems = self.iter();
+
+ match elems.next() {
+ Some(&elem) => elems.fold(elem, |a, &b| a.min(b)),
+ // NB `unreachable!` because `Sample` is guaranteed to have at least one data point
+ None => unreachable!(),
+ }
+ }
+
+ /// Returns a "view" into the percentiles of the sample
+ ///
+ /// This "view" makes consecutive computations of percentiles much faster (`O(1)`)
+ ///
+ /// - Time: `O(N log N) where N = length`
+ /// - Memory: `O(length)`
+ pub fn percentiles(&self) -> Percentiles<A>
+ where
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+ {
+ use std::cmp::Ordering;
+
+ // NB This function assumes that there are no `NaN`s in the sample
+ fn cmp<T>(a: &T, b: &T) -> Ordering
+ where
+ T: PartialOrd,
+ {
+ match a.partial_cmp(b) {
+ Some(o) => o,
+ // Arbitrary way to handle NaNs that should never happen
+ None => Ordering::Equal,
+ }
+ }
+
+ let mut v = self.to_vec().into_boxed_slice();
+ v.par_sort_unstable_by(cmp);
+
+ // NB :-1: to intra-crate privacy rules
+ unsafe { mem::transmute(v) }
+ }
+
+ /// Returns the standard deviation of the sample
+ ///
+ /// The `mean` can be optionally passed along to speed up (2X) the computation
+ ///
+ /// - Time: `O(length)`
+ pub fn std_dev(&self, mean: Option<A>) -> A {
+ self.var(mean).sqrt()
+ }
+
+ /// Returns the standard deviation as a percentage of the mean
+ ///
+ /// - Time: `O(length)`
+ pub fn std_dev_pct(&self) -> A {
+ let _100 = A::cast(100);
+ let mean = self.mean();
+ let std_dev = self.std_dev(Some(mean));
+
+ (std_dev / mean) * _100
+ }
+
+ /// Returns the sum of all the elements of the sample
+ ///
+ /// - Time: `O(length)`
+ pub fn sum(&self) -> A {
+ crate::stats::sum(self)
+ }
+
+ /// Returns the t score between these two samples
+ ///
+ /// - Time: `O(length)`
+ pub fn t(&self, other: &Sample<A>) -> A {
+ let (x_bar, y_bar) = (self.mean(), other.mean());
+ let (s2_x, s2_y) = (self.var(Some(x_bar)), other.var(Some(y_bar)));
+ let n_x = A::cast(self.len());
+ let n_y = A::cast(other.len());
+ let num = x_bar - y_bar;
+ let den = (s2_x / n_x + s2_y / n_y).sqrt();
+
+ num / den
+ }
+
+ /// Returns the variance of the sample
+ ///
+ /// The `mean` can be optionally passed along to speed up (2X) the computation
+ ///
+ /// - Time: `O(length)`
+ pub fn var(&self, mean: Option<A>) -> A {
+ use std::ops::Add;
+
+ let mean = mean.unwrap_or_else(|| self.mean());
+ let slice = self;
+
+ let sum = slice
+ .iter()
+ .map(|&x| (x - mean).powi(2))
+ .fold(A::cast(0), Add::add);
+
+ sum / A::cast(slice.len() - 1)
+ }
+
+ // TODO Remove the `T` parameter in favor of `S::Output`
+ /// Returns the bootstrap distributions of the parameters estimated by the 1-sample statistic
+ ///
+ /// - Multi-threaded
+ /// - Time: `O(nresamples)`
+ /// - Memory: `O(nresamples)`
+ pub fn bootstrap<T, S>(&self, nresamples: usize, statistic: S) -> T::Distributions
+ where
+ S: Fn(&Sample<A>) -> T + Sync,
+ T: Tuple + Send,
+ T::Distributions: Send,
+ T::Builder: Send,
+ {
+ (0..nresamples)
+ .into_par_iter()
+ .map_init(
+ || Resamples::new(self),
+ |resamples, _| statistic(resamples.next()),
+ )
+ .fold(
+ || T::Builder::new(0),
+ |mut sub_distributions, sample| {
+ sub_distributions.push(sample);
+ sub_distributions
+ },
+ )
+ .reduce(
+ || T::Builder::new(0),
+ |mut a, mut b| {
+ a.extend(&mut b);
+ a
+ },
+ )
+ .complete()
+ }
+
+ #[cfg(test)]
+ pub fn iqr(&self) -> A
+ where
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+ {
+ self.percentiles().iqr()
+ }
+
+ #[cfg(test)]
+ pub fn median(&self) -> A
+ where
+ usize: cast::From<A, Output = Result<usize, cast::Error>>,
+ {
+ self.percentiles().median()
+ }
+}
+
+impl<A> ops::Deref for Sample<A> {
+ type Target = [A];
+
+ fn deref(&self) -> &[A] {
+ &self.0
+ }
+}