From a5741100482eb50bb00ac0f8d22198277cd41fa5 Mon Sep 17 00:00:00 2001 From: Alastair Murray Date: Tue, 9 Feb 2016 13:08:56 +0000 Subject: Close a memory leak in configuration code This was causing configuration failures on address sanitizer builds. --- cmake/posix_regex.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/posix_regex.cpp b/cmake/posix_regex.cpp index a31af80..466dc62 100644 --- a/cmake/posix_regex.cpp +++ b/cmake/posix_regex.cpp @@ -7,6 +7,8 @@ int main() { if (ec != 0) { return ec; } - return regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0; + int ret = regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0; + regfree(&re); + return ret; } -- cgit v1.2.3 From 53aca9bc7128b7c0cd513163c5432481f12025ee Mon Sep 17 00:00:00 2001 From: Mohamed Amin JABRI Date: Wed, 24 Feb 2016 17:36:34 +0900 Subject: Pass const State to Fixture::TearDown. Fix memory leak in fixture_test --- include/benchmark/benchmark_api.h | 4 ++-- test/fixture_test.cc | 23 +++++++++++++++++------ test/map_test.cc | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 0c5115d..7a42025 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -491,11 +491,11 @@ public: virtual void Run(State& st) { this->SetUp(st); this->BenchmarkCase(st); - this->TearDown(); + this->TearDown(st); } virtual void SetUp(const State&) {} - virtual void TearDown() {} + virtual void TearDown(const State&) {} protected: virtual void BenchmarkCase(State&) = 0; diff --git a/test/fixture_test.cc b/test/fixture_test.cc index 92fbc4c..bf800fd 100644 --- a/test/fixture_test.cc +++ b/test/fixture_test.cc @@ -6,14 +6,18 @@ class MyFixture : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State&) { - assert(data.get() == nullptr); - data.reset(new int(42)); + void SetUp(const ::benchmark::State& state) { + if (state.thread_index == 0) { + assert(data.get() == nullptr); + data.reset(new int(42)); + } } - void TearDown() { - assert(data.get() != nullptr); - data.release(); + void TearDown(const ::benchmark::State& state) { + if (state.thread_index == 0) { + assert(data.get() != nullptr); + data.reset(); + } } ~MyFixture() { @@ -32,10 +36,17 @@ BENCHMARK_F(MyFixture, Foo)(benchmark::State& st) { } BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) { + if (st.thread_index == 0) { + assert(data.get() != nullptr); + assert(*data == 42); + } while (st.KeepRunning()) { + assert(data.get() != nullptr); + assert(*data == 42); } st.SetItemsProcessed(st.range_x()); } BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42); +BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42)->ThreadPerCpu(); BENCHMARK_MAIN() diff --git a/test/map_test.cc b/test/map_test.cc index 8d5f6ec..58399c1 100644 --- a/test/map_test.cc +++ b/test/map_test.cc @@ -36,7 +36,7 @@ class MapFixture : public ::benchmark::Fixture { m = ConstructRandomMap(st.range_x()); } - void TearDown() { + void TearDown(const ::benchmark::State&) { m.clear(); } -- cgit v1.2.3 From 0dca953b135d967d1a8d3271718d9729ec026280 Mon Sep 17 00:00:00 2001 From: Ian Henriksen Date: Wed, 24 Feb 2016 12:31:31 -0700 Subject: Include cstdlib in map_test to make sure std::rand is available when building with libc++. --- test/map_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/map_test.cc b/test/map_test.cc index 58399c1..5eccf8d 100644 --- a/test/map_test.cc +++ b/test/map_test.cc @@ -1,5 +1,6 @@ #include "benchmark/benchmark.h" +#include #include namespace { -- cgit v1.2.3 From cded70a1660e81f854b5d41795a514ec0825c32a Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Thu, 24 Mar 2016 22:18:55 +0100 Subject: Add optional ms time unit for console reporter Some benchmarks may run a few milliseconds which makes it kind of hard to visually compare, since the currently only available nanoseconds numbers can get very large in this case. Therefore this commit adds an optional command line flag --benchmark_time_unit which lets the user choose between ns and ms time units for displaying the mean execution time. --- include/benchmark/reporter.h | 3 +++ src/benchmark.cc | 19 +++++++++++++++++-- src/console_reporter.cc | 9 +++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index d23ab65..f2a8dc2 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -36,6 +36,8 @@ class BenchmarkReporter { // The number of chars in the longest benchmark name. size_t name_field_width; + // The time unit for displayed execution time. + std::string time_unit; }; struct Run { @@ -94,6 +96,7 @@ protected: virtual void PrintRunData(const Run& report); size_t name_field_width_; + std::string time_unit_; }; class JSONReporter : public BenchmarkReporter { diff --git a/src/benchmark.cc b/src/benchmark.cc index 08b180e..dd37202 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -64,6 +64,10 @@ DEFINE_int32(benchmark_repetitions, 1, "The number of runs of each benchmark. If greater than 1, the " "mean and standard deviation of the runs will be reported."); +DEFINE_string(benchmark_time_unit, "ns", + "The time unit to use for console output. Valid values are " + "'ns', or 'ms'."); + DEFINE_string(benchmark_format, "tabular", "The format to use for console output. Valid values are " "'tabular', 'json', or 'csv'."); @@ -779,7 +783,7 @@ void PrintBenchmarkList() { } } -void RunMatchingBenchmarks(const std::string& spec, +void RunMatchingBenchmarks(const std::string& spec, const std::string& timeUnit, BenchmarkReporter* reporter) { CHECK(reporter != nullptr); if (spec.empty()) return; @@ -804,6 +808,7 @@ void RunMatchingBenchmarks(const std::string& spec, context.cpu_scaling_enabled = CpuScalingEnabled(); context.name_field_width = name_field_width; + context.time_unit = timeUnit; if (reporter->ReportContext(context)) { for (const auto& benchmark : benchmarks) { @@ -838,6 +843,7 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { internal::PrintBenchmarkList(); return; } + std::string timeUnit = FLAGS_benchmark_time_unit; std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") spec = "."; // Regexp that matches all benchmarks @@ -847,7 +853,7 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { default_reporter = internal::GetDefaultReporter(); reporter = default_reporter.get(); } - internal::RunMatchingBenchmarks(spec, reporter); + internal::RunMatchingBenchmarks(spec, timeUnit, reporter); reporter->Finalize(); } @@ -860,6 +866,7 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" + " [--benchmark_time_unit=]\n" " [--benchmark_format=]\n" " [--color_print={true|false}]\n" " [--v=]\n"); @@ -878,6 +885,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_min_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || + ParseStringFlag(argv[i], "benchmark_time_unit", + &FLAGS_benchmark_time_unit) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || ParseBoolFlag(argv[i], "color_print", @@ -891,6 +900,12 @@ void ParseCommandLineFlags(int* argc, char** argv) { PrintUsageAndExit(); } } + + if (FLAGS_benchmark_time_unit != "ns" && + FLAGS_benchmark_time_unit != "ms") { + PrintUsageAndExit(); + } + if (FLAGS_benchmark_format != "tabular" && FLAGS_benchmark_format != "json" && FLAGS_benchmark_format != "csv") { diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 092936d..6af5157 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -29,6 +29,7 @@ namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; + time_unit_ = context.time_unit; std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; @@ -46,9 +47,11 @@ bool ConsoleReporter::ReportContext(const Context& context) { "affected.\n"; #endif + std::string timeLabel = "Time(" + time_unit_ + ")"; + std::string cpuLabel = "CPU(" + time_unit_ + ")"; int output_width = fprintf(stdout, "%-*s %10s %10s %10s\n", static_cast(name_field_width_), "Benchmark", - "Time(ns)", "CPU(ns)", "Iterations"); + timeLabel.c_str(), cpuLabel.c_str(), "Iterations"); std::cout << std::string(output_width - 1, '-') << "\n"; return true; @@ -92,7 +95,9 @@ void ConsoleReporter::PrintRunData(const Run& result) { " items/s"); } - double const multiplier = 1e9; // nano second multiplier + double const multiplier = time_unit_ == "ns" ? 1e9 : 1e3; // nano second or + // millis multiplier + ColorPrintf(COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if (result.iterations == 0) { -- cgit v1.2.3 From 3a02c462c795fd60e2620fc78c6f8c5b63aa1764 Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Thu, 24 Mar 2016 22:34:23 +0100 Subject: Add myself to the contributors list --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ed55bcf..2bbe19e 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -32,6 +32,7 @@ Evgeny Safronov Felix Homann JianXiong Zhou Kaito Udagawa +Kai Wolf Lei Xu Matt Clarkson Oleksandr Sochka -- cgit v1.2.3 From 7c69b36078b5773fbd6b09b539a30400138607a7 Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Fri, 25 Mar 2016 22:47:27 +0100 Subject: Add an additional parameter for time units --- include/benchmark/benchmark_api.h | 11 +++++++++++ include/benchmark/reporter.h | 13 +++++++++---- src/benchmark.cc | 34 +++++++++++++++++----------------- src/console_reporter.cc | 37 ++++++++++++++++++++++++++----------- test/CMakeLists.txt | 3 +++ test/options_test.cc | 11 +++++++++++ 6 files changed, 77 insertions(+), 32 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 7a42025..251fd59 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -216,6 +216,13 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { } #endif +// TimeUnit is passed to a benchmark in order to specify the order of magnitude +// for the measured time. +enum TimeUnit { + kNanosecond, + kMicrosecond, + kMillisecond +}; // State is passed to a running Benchmark and contains state for the // benchmark to use. @@ -390,6 +397,9 @@ public: // REQUIRES: The function passed to the constructor must accept an arg1. Benchmark* Arg(int x); + // Run this benchmark with the given time unit for the generated output report + Benchmark* Unit(TimeUnit unit); + // Run this benchmark once for a number of values picked from the // range [start..limit]. (start and limit are always picked.) // REQUIRES: The function passed to the constructor must accept an arg1. @@ -534,6 +544,7 @@ protected: // Old-style macros #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) #define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->ArgPair((a1), (a2)) +#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) #define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) #define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ BENCHMARK(n)->RangePair((l1), (h1), (l2), (h2)) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index f2a8dc2..9c4a69d 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -36,13 +36,12 @@ class BenchmarkReporter { // The number of chars in the longest benchmark name. size_t name_field_width; - // The time unit for displayed execution time. - std::string time_unit; }; struct Run { Run() : iterations(1), + time_unit(kNanosecond), real_accumulated_time(0), cpu_accumulated_time(0), bytes_per_second(0), @@ -52,6 +51,7 @@ class BenchmarkReporter { std::string benchmark_name; std::string report_label; // Empty if not set by benchmark. int64_t iterations; + TimeUnit time_unit; double real_accumulated_time; double cpu_accumulated_time; @@ -86,17 +86,22 @@ protected: static void ComputeStats(std::vector const& reports, Run* mean, Run* stddev); }; +typedef std::pair TimeUnitMultiplier; + // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). class ConsoleReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); -protected: + + protected: virtual void PrintRunData(const Run& report); + private: + TimeUnitMultiplier getTimeUnitAndMultiplier(TimeUnit unit); + size_t name_field_width_; - std::string time_unit_; }; class JSONReporter : public BenchmarkReporter { diff --git a/src/benchmark.cc b/src/benchmark.cc index dd37202..cb55f96 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -64,10 +64,6 @@ DEFINE_int32(benchmark_repetitions, 1, "The number of runs of each benchmark. If greater than 1, the " "mean and standard deviation of the runs will be reported."); -DEFINE_string(benchmark_time_unit, "ns", - "The time unit to use for console output. Valid values are " - "'ns', or 'ms'."); - DEFINE_string(benchmark_format, "tabular", "The format to use for console output. Valid values are " "'tabular', 'json', or 'csv'."); @@ -265,6 +261,7 @@ struct Benchmark::Instance { int arg1; bool has_arg2; int arg2; + TimeUnit time_unit; bool use_real_time; double min_time; int threads; // Number of concurrent threads to use @@ -298,6 +295,7 @@ public: ~BenchmarkImp(); void Arg(int x); + void Unit(TimeUnit unit); void Range(int start, int limit); void DenseRange(int start, int limit); void ArgPair(int start, int limit); @@ -317,6 +315,7 @@ private: std::string name_; int arg_count_; std::vector< std::pair > args_; // Args for all benchmark runs + TimeUnit time_unit_; double min_time_; bool use_real_time_; std::vector thread_counts_; @@ -376,6 +375,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.arg1 = args.first; instance.has_arg2 = family->arg_count_ == 2; instance.arg2 = args.second; + instance.time_unit = family->time_unit_; instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; instance.threads = num_threads; @@ -410,7 +410,7 @@ bool BenchmarkFamilies::FindBenchmarks( } BenchmarkImp::BenchmarkImp(const char* name) - : name_(name), arg_count_(-1), + : name_(name), arg_count_(-1), time_unit_(kNanosecond), min_time_(0.0), use_real_time_(false) { } @@ -423,6 +423,10 @@ void BenchmarkImp::Arg(int x) { args_.emplace_back(x, -1); } +void BenchmarkImp::Unit(TimeUnit unit) { + time_unit_ = unit; +} + void BenchmarkImp::Range(int start, int limit) { CHECK(arg_count_ == -1 || arg_count_ == 1); arg_count_ = 1; @@ -535,6 +539,11 @@ Benchmark* Benchmark::Arg(int x) { return this; } +Benchmark* Benchmark::Unit(TimeUnit unit) { + imp_->Unit(unit); + return this; +} + Benchmark* Benchmark::Range(int start, int limit) { imp_->Range(start, limit); return this; @@ -703,6 +712,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.report_label = label; // Report the total iterations across all threads. report.iterations = static_cast(iters) * b.threads; + report.time_unit = b.time_unit; report.real_accumulated_time = real_accumulated_time; report.cpu_accumulated_time = cpu_accumulated_time; report.bytes_per_second = bytes_per_second; @@ -783,7 +793,7 @@ void PrintBenchmarkList() { } } -void RunMatchingBenchmarks(const std::string& spec, const std::string& timeUnit, +void RunMatchingBenchmarks(const std::string& spec, BenchmarkReporter* reporter) { CHECK(reporter != nullptr); if (spec.empty()) return; @@ -808,7 +818,6 @@ void RunMatchingBenchmarks(const std::string& spec, const std::string& timeUnit, context.cpu_scaling_enabled = CpuScalingEnabled(); context.name_field_width = name_field_width; - context.time_unit = timeUnit; if (reporter->ReportContext(context)) { for (const auto& benchmark : benchmarks) { @@ -843,7 +852,6 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { internal::PrintBenchmarkList(); return; } - std::string timeUnit = FLAGS_benchmark_time_unit; std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") spec = "."; // Regexp that matches all benchmarks @@ -853,7 +861,7 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { default_reporter = internal::GetDefaultReporter(); reporter = default_reporter.get(); } - internal::RunMatchingBenchmarks(spec, timeUnit, reporter); + internal::RunMatchingBenchmarks(spec, reporter); reporter->Finalize(); } @@ -866,7 +874,6 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" - " [--benchmark_time_unit=]\n" " [--benchmark_format=]\n" " [--color_print={true|false}]\n" " [--v=]\n"); @@ -885,8 +892,6 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_min_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || - ParseStringFlag(argv[i], "benchmark_time_unit", - &FLAGS_benchmark_time_unit) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || ParseBoolFlag(argv[i], "color_print", @@ -901,11 +906,6 @@ void ParseCommandLineFlags(int* argc, char** argv) { } } - if (FLAGS_benchmark_time_unit != "ns" && - FLAGS_benchmark_time_unit != "ms") { - PrintUsageAndExit(); - } - if (FLAGS_benchmark_format != "tabular" && FLAGS_benchmark_format != "json" && FLAGS_benchmark_format != "csv") { diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 6af5157..c07ed5a 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "check.h" @@ -29,7 +30,6 @@ namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; - time_unit_ = context.time_unit; std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; @@ -47,11 +47,9 @@ bool ConsoleReporter::ReportContext(const Context& context) { "affected.\n"; #endif - std::string timeLabel = "Time(" + time_unit_ + ")"; - std::string cpuLabel = "CPU(" + time_unit_ + ")"; - int output_width = fprintf(stdout, "%-*s %10s %10s %10s\n", + int output_width = fprintf(stdout, "%-*s %13s %13s %10s\n", static_cast(name_field_width_), "Benchmark", - timeLabel.c_str(), cpuLabel.c_str(), "Iterations"); + "Time", "CPU", "Iterations"); std::cout << std::string(output_width - 1, '-') << "\n"; return true; @@ -95,21 +93,26 @@ void ConsoleReporter::PrintRunData(const Run& result) { " items/s"); } - double const multiplier = time_unit_ == "ns" ? 1e9 : 1e3; // nano second or - // millis multiplier + double multiplier; + const char* timeLabel; + std::tie(timeLabel, multiplier) = getTimeUnitAndMultiplier(result.time_unit); ColorPrintf(COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if (result.iterations == 0) { - ColorPrintf(COLOR_YELLOW, "%10.0f %10.0f ", + ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, - result.cpu_accumulated_time * multiplier); + timeLabel, + result.cpu_accumulated_time * multiplier, + timeLabel); } else { - ColorPrintf(COLOR_YELLOW, "%10.0f %10.0f ", + ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", (result.real_accumulated_time * multiplier) / (static_cast(result.iterations)), + timeLabel, (result.cpu_accumulated_time * multiplier) / - (static_cast(result.iterations))); + (static_cast(result.iterations)), + timeLabel); } ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); ColorPrintf(COLOR_DEFAULT, "%*s %*s %s\n", @@ -118,4 +121,16 @@ void ConsoleReporter::PrintRunData(const Run& result) { result.report_label.c_str()); } +TimeUnitMultiplier ConsoleReporter::getTimeUnitAndMultiplier(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return std::make_pair("ms", 1e3); + case kMicrosecond: + return std::make_pair("us", 1e6); + case kNanosecond: + default: + return std::make_pair("ns", 1e9); + } +} + } // end namespace benchmark diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a10a53a..196c0ed 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,5 +1,8 @@ # Enable the tests +# Allow the source files to find headers in src/ +include_directories(${PROJECT_SOURCE_DIR}/src) + find_package(Threads REQUIRED) set(CXX03_FLAGS "${CMAKE_CXX_FLAGS}") diff --git a/test/options_test.cc b/test/options_test.cc index d4c682d..47563fa 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -1,11 +1,22 @@ #include "benchmark/benchmark_api.h" +#include "sleep.h" void BM_basic(benchmark::State& state) { while (state.KeepRunning()) { } } + +void BM_basic_slow(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::SleepForMilliseconds(state.range_x()); + } +} + BENCHMARK(BM_basic); BENCHMARK(BM_basic)->Arg(42); +BENCHMARK(BM_basic_slow)->Arg(10)->Unit(benchmark::kNanosecond); +BENCHMARK(BM_basic_slow)->Arg(100)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kMillisecond); BENCHMARK(BM_basic)->Range(1, 8); BENCHMARK(BM_basic)->DenseRange(10, 15); BENCHMARK(BM_basic)->ArgPair(42, 42); -- cgit v1.2.3 From 0b4111c3b31db8806e0c3960c7f1f541b20cdb8b Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Mon, 28 Mar 2016 21:32:11 +0200 Subject: Refactor GetTimeUnitAndMultiplier and add example --- README.md | 8 ++++++++ include/benchmark/benchmark_api.h | 7 +++++++ include/benchmark/reporter.h | 10 ++++------ src/console_reporter.cc | 14 +------------- src/reporter.cc | 12 ++++++++++++ 5 files changed, 32 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 21ae478..b4c6f7c 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,14 @@ static void BM_test(benchmark::State& state) { } ``` +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: + +```c++ +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); +``` + Benchmark Fixtures ------------------ Fixture tests are created by diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 251fd59..f744015 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -137,6 +137,13 @@ static void BM_MultiThreaded(benchmark::State& state) { } } BENCHMARK(BM_MultiThreaded)->Threads(4); + + +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: + +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); */ #ifndef BENCHMARK_BENCHMARK_API_H_ diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 9c4a69d..aaf5fbf 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -22,6 +22,8 @@ namespace benchmark { +typedef std::pair TimeUnitMultiplier; + // Interface for custom benchmark result printers. // By default, benchmark reports are printed to stdout. However an application // can control the destination of the reports by calling @@ -83,11 +85,10 @@ class BenchmarkReporter { virtual ~BenchmarkReporter(); protected: - static void ComputeStats(std::vector const& reports, Run* mean, Run* stddev); + static void ComputeStats(std::vector const& reports, Run* mean, Run* stddev); + static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); }; -typedef std::pair TimeUnitMultiplier; - // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). class ConsoleReporter : public BenchmarkReporter { @@ -98,9 +99,6 @@ class ConsoleReporter : public BenchmarkReporter { protected: virtual void PrintRunData(const Run& report); - private: - TimeUnitMultiplier getTimeUnitAndMultiplier(TimeUnit unit); - size_t name_field_width_; }; diff --git a/src/console_reporter.cc b/src/console_reporter.cc index c07ed5a..375e861 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -95,7 +95,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { double multiplier; const char* timeLabel; - std::tie(timeLabel, multiplier) = getTimeUnitAndMultiplier(result.time_unit); + std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(result.time_unit); ColorPrintf(COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); @@ -121,16 +121,4 @@ void ConsoleReporter::PrintRunData(const Run& result) { result.report_label.c_str()); } -TimeUnitMultiplier ConsoleReporter::getTimeUnitAndMultiplier(TimeUnit unit) { - switch (unit) { - case kMillisecond: - return std::make_pair("ms", 1e3); - case kMicrosecond: - return std::make_pair("us", 1e6); - case kNanosecond: - default: - return std::make_pair("ns", 1e9); - } -} - } // end namespace benchmark diff --git a/src/reporter.cc b/src/reporter.cc index 4b47e3d..036546e 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -77,6 +77,18 @@ void BenchmarkReporter::ComputeStats( stddev_data->items_per_second = items_per_second_stat.StdDev(); } +TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return std::make_pair("ms", 1e3); + case kMicrosecond: + return std::make_pair("us", 1e6); + case kNanosecond: + default: + return std::make_pair("ns", 1e9); + } +} + void BenchmarkReporter::Finalize() { } -- cgit v1.2.3 From e6d62fd135d7c1c621ff3b25056bd5ea4d1754ec Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Tue, 29 Mar 2016 20:35:38 +0200 Subject: Add GetTimeAndMultiplier to json and csv reporter as well --- src/csv_reporter.cc | 8 ++++++-- src/json_reporter.cc | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index d78a9df..3284ba8 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -42,7 +42,7 @@ bool CSVReporter::ReportContext(const Context& context) { std::cerr << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; #endif - std::cout << "name,iterations,real_time,cpu_time,bytes_per_second," + std::cout << "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "items_per_second,label\n"; return true; } @@ -66,7 +66,10 @@ void CSVReporter::ReportRuns(std::vector const& reports) { } void CSVReporter::PrintRunData(Run const& run) { - double const multiplier = 1e9; // nano second multiplier + double multiplier; + const char* timeLabel; + std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); + double cpu_time = run.cpu_accumulated_time * multiplier; double real_time = run.real_accumulated_time * multiplier; if (run.iterations != 0) { @@ -83,6 +86,7 @@ void CSVReporter::PrintRunData(Run const& run) { std::cout << run.iterations << ","; std::cout << real_time << ","; std::cout << cpu_time << ","; + std::cout << timeLabel << ","; if (run.bytes_per_second > 0.0) { std::cout << run.bytes_per_second; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index def50ac..b0198fb 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -120,7 +120,10 @@ void JSONReporter::Finalize() { } void JSONReporter::PrintRunData(Run const& run) { - double const multiplier = 1e9; // nano second multiplier + double multiplier; + const char* timeLabel; + std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); + double cpu_time = run.cpu_accumulated_time * multiplier; double real_time = run.real_accumulated_time * multiplier; if (run.iterations != 0) { @@ -140,7 +143,10 @@ void JSONReporter::PrintRunData(Run const& run) { << FormatKV("real_time", RoundDouble(real_time)) << ",\n"; out << indent - << FormatKV("cpu_time", RoundDouble(cpu_time)); + << FormatKV("cpu_time", RoundDouble(cpu_time)) + << ",\n"; + out << indent + << FormatKV("time_unit", timeLabel); if (run.bytes_per_second > 0.0) { out << ",\n" << indent << FormatKV("bytes_per_second", RoundDouble(run.bytes_per_second)); -- cgit v1.2.3 From 1043f45bedb5afcd982f4a4b4be716bcabb286f4 Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Tue, 29 Mar 2016 22:10:07 +0200 Subject: Add explanatory comment why src headers are needed in test/CMakeLists.txt --- test/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 196c0ed..31d62e8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,7 @@ # Enable the tests -# Allow the source files to find headers in src/ +# Allow the test files to find headers in src/ since we rely on +# SleepForMilliseconds(int milliseconds) in options_test.cc include_directories(${PROJECT_SOURCE_DIR}/src) find_package(Threads REQUIRED) -- cgit v1.2.3 From 1203b3cbe47ad772291fe520efb2a029687229ed Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Wed, 30 Mar 2016 09:14:04 +0200 Subject: Fix missing header in csv/json_reporter --- src/csv_reporter.cc | 1 + src/json_reporter.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 3284ba8..3f67d1d 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "string_util.h" diff --git a/src/json_reporter.cc b/src/json_reporter.cc index b0198fb..7ed141f 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "string_util.h" -- cgit v1.2.3 From 838719dc02d1104153fb4193acea32a64d5559e7 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 18 Apr 2016 14:24:13 +0200 Subject: Update benchmark_api.h --- include/benchmark/benchmark_api.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 7a42025..8247be3 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -305,10 +305,10 @@ public: // If this routine is called, the specified label is printed at the // end of the benchmark report line for the currently executing // benchmark. Example: - // static void BM_Compress(int iters) { + // static void BM_Compress(benchmark::State& state) { // ... // double compress = input_size / output_size; - // benchmark::SetLabel(StringPrintf("compress:%.1f%%", 100.0*compression)); + // state.SetLabel(StringPrintf("compress:%.1f%%", 100.0*compression)); // } // Produces output that looks like: // BM_Compress 50 50 14115038 compress:27.3% -- cgit v1.2.3 From d6f96ed6399bd943be7592e1114bf0ddf64d68b0 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 19 Apr 2016 09:34:13 -0700 Subject: Add section on iterations. Also add some subheadings, and fix up the line lengths. Fixes #194 --- README.md | 86 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 21ae478..7d2cc17 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -benchmark -========= +# benchmark [![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) @@ -10,10 +9,9 @@ Discussion group: https://groups.google.com/d/forum/benchmark-discuss IRC channel: https://freenode.net #googlebenchmark -Example usage -------------- -Define a function that executes the code to be measured a -specified number of times: +## Example usage +### Basic usage +Define a function that executes the code to be measured. ```c++ static void BM_StringCreation(benchmark::State& state) { @@ -34,15 +32,16 @@ BENCHMARK(BM_StringCopy); BENCHMARK_MAIN(); ``` -Sometimes a family of microbenchmarks can be implemented with -just one routine that takes an extra argument to specify which -one of the family of benchmarks to run. For example, the following -code defines a family of microbenchmarks for measuring the speed -of `memcpy()` calls of different lengths: +### Passing arguments +Sometimes a family of benchmarks can be implemented with just one routine that +takes an extra argument to specify which one of the family of benchmarks to +run. For example, the following code defines a family of benchmarks for +measuring the speed of `memcpy()` calls of different lengths: ```c++ static void BM_memcpy(benchmark::State& state) { - char* src = new char[state.range_x()]; char* dst = new char[state.range_x()]; + char* src = new char[state.range_x()]; + char* dst = new char[state.range_x()]; memset(src, 'x', state.range_x()); while (state.KeepRunning()) memcpy(dst, src, state.range_x()); @@ -54,18 +53,17 @@ static void BM_memcpy(benchmark::State& state) { BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); ``` -The preceding code is quite repetitive, and can be replaced with the -following short-hand. The following invocation will pick a few -appropriate arguments in the specified range and will generate a -microbenchmark for each such argument. +The preceding code is quite repetitive, and can be replaced with the following +short-hand. The following invocation will pick a few appropriate arguments in +the specified range and will generate a benchmark for each such argument. ```c++ BENCHMARK(BM_memcpy)->Range(8, 8<<10); ``` -You might have a microbenchmark that depends on two inputs. For -example, the following code defines a family of microbenchmarks for -measuring the speed of set insertion. +You might have a benchmark that depends on two inputs. For example, the +following code defines a family of benchmarks for measuring the speed of set +insertion. ```c++ static void BM_SetInsert(benchmark::State& state) { @@ -88,19 +86,18 @@ BENCHMARK(BM_SetInsert) ->ArgPair(8<<10, 512); ``` -The preceding code is quite repetitive, and can be replaced with -the following short-hand. The following macro will pick a few -appropriate arguments in the product of the two specified ranges -and will generate a microbenchmark for each such pair. +The preceding code is quite repetitive, and can be replaced with the following +short-hand. The following macro will pick a few appropriate arguments in the +product of the two specified ranges and will generate a benchmark for each such +pair. ```c++ BENCHMARK(BM_SetInsert)->RangePair(1<<10, 8<<10, 1, 512); ``` -For more complex patterns of inputs, passing a custom function -to Apply allows programmatic specification of an -arbitrary set of arguments to run the microbenchmark on. -The following example enumerates a dense range on one parameter, +For more complex patterns of inputs, passing a custom function to `Apply` allows +programmatic specification of an arbitrary set of arguments on which to run the +benchmark. The following example enumerates a dense range on one parameter, and a sparse range on the second. ```c++ @@ -112,9 +109,10 @@ static void CustomArguments(benchmark::internal::Benchmark* b) { BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` -Templated microbenchmarks work the same way: -Produce then consume 'size' messages 'iters' times -Measures throughput in the absence of multiprogramming. +### Templated benchmarks +Templated benchmarks work the same way: This example produces and consumes +messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the +absence of multiprogramming. ```c++ template int BM_Sequential(benchmark::State& state) { @@ -145,11 +143,12 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` +### Multithreaded benchmarks In a multithreaded test (benchmark invoked by multiple threads simultaneously), it is guaranteed that none of the threads will start until all have called -KeepRunning, and all will have finished before KeepRunning returns false. As -such, any global setup or teardown you want to do can be -wrapped in a check against the thread index: +`KeepRunning`, and all will have finished before KeepRunning returns false. As +such, any global setup or teardown can be wrapped in a check against the thread +index: ```c++ static void BM_MultiThreaded(benchmark::State& state) { @@ -176,6 +175,7 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. +### Preventing optimisation To prevent a value or expression from being optimized away by the compiler the `benchmark::DoNotOptimize(...)` function can be used. @@ -190,8 +190,15 @@ static void BM_test(benchmark::State& state) { } ``` -Benchmark Fixtures ------------------- +## Controlling number of iterations +In all cases, the number of iterations for which the benchmark is run is +governed by the amount of time the benchmark takes. Concretely, the number of +iterations is at least one, not more than 1e9, until CPU time is greater than +the minimum time, or the wallclock time is 5x minimum time. The minimum time is +set as a flag `--benchmark_min_time` or per benchmark by calling `MinTime` on +the registered benchmark object. + +## Fixtures Fixture tests are created by first defining a type that derives from ::benchmark::Fixture and then creating/registering the tests using the following macros: @@ -221,8 +228,7 @@ BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); /* BarTest is now registered */ ``` -Output Formats --------------- +## Output Formats The library supports multiple output formats. Use the `--benchmark_format=` flag to set the format type. `tabular` is the default format. @@ -290,8 +296,7 @@ name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label "BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, ``` -Debug vs Release ----------------- +## Debug vs Release By default, benchmark builds as a debug library. You will see a warning in the output when this is the case. To build it as a release library instead, use: ``` @@ -304,6 +309,5 @@ To enable link-time optimisation, use cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true ``` -Linking against the library ---------------------------- +## Linking against the library When using gcc, it is necessary to link against pthread to avoid runtime exceptions. This is due to how gcc implements std::thread. See [issue #67](https://github.com/google/benchmark/issues/67) for more details. -- cgit v1.2.3 From fb733897c5eea5b9a04ebed5cfae37d608262129 Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Mon, 2 May 2016 19:53:45 +0200 Subject: Remove sleep.h dependency for tests --- test/CMakeLists.txt | 4 ---- test/options_test.cc | 12 ++++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 31d62e8..a10a53a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,9 +1,5 @@ # Enable the tests -# Allow the test files to find headers in src/ since we rely on -# SleepForMilliseconds(int milliseconds) in options_test.cc -include_directories(${PROJECT_SOURCE_DIR}/src) - find_package(Threads REQUIRED) set(CXX03_FLAGS "${CMAKE_CXX_FLAGS}") diff --git a/test/options_test.cc b/test/options_test.cc index 47563fa..4fe2d17 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -1,5 +1,7 @@ #include "benchmark/benchmark_api.h" -#include "sleep.h" + +#include +#include void BM_basic(benchmark::State& state) { while (state.KeepRunning()) { @@ -7,8 +9,14 @@ void BM_basic(benchmark::State& state) { } void BM_basic_slow(benchmark::State& state) { + + int milliseconds = state.range_x(); + std::chrono::duration sleep_duration { + static_cast(milliseconds) + }; + while (state.KeepRunning()) { - benchmark::SleepForMilliseconds(state.range_x()); + std::this_thread::sleep_for(sleep_duration); } } -- cgit v1.2.3 From 02f409a71f5231bf9b8803e771bdfa1056da23cc Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 2 May 2016 12:04:16 -0700 Subject: Only output optional fields if they're set --- src/benchmark.cc | 10 +++++----- src/console_reporter.cc | 20 ++++++++++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index cb55f96..35b5e2d 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -64,9 +64,9 @@ DEFINE_int32(benchmark_repetitions, 1, "The number of runs of each benchmark. If greater than 1, the " "mean and standard deviation of the runs will be reported."); -DEFINE_string(benchmark_format, "tabular", +DEFINE_string(benchmark_format, "console", "The format to use for console output. Valid values are " - "'tabular', 'json', or 'csv'."); + "'console', 'json', or 'csv'."); DEFINE_bool(color_print, true, "Enables colorized logging."); @@ -828,7 +828,7 @@ void RunMatchingBenchmarks(const std::string& spec, std::unique_ptr GetDefaultReporter() { typedef std::unique_ptr PtrType; - if (FLAGS_benchmark_format == "tabular") { + if (FLAGS_benchmark_format == "console") { return PtrType(new ConsoleReporter); } else if (FLAGS_benchmark_format == "json") { return PtrType(new JSONReporter); @@ -874,7 +874,7 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" - " [--benchmark_format=]\n" + " [--benchmark_format=]\n" " [--color_print={true|false}]\n" " [--v=]\n"); exit(0); @@ -906,7 +906,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { } } - if (FLAGS_benchmark_format != "tabular" && + if (FLAGS_benchmark_format != "console" && FLAGS_benchmark_format != "json" && FLAGS_benchmark_format != "csv") { PrintUsageAndExit(); diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 375e861..56bd3ce 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -99,6 +99,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { ColorPrintf(COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); + if (result.iterations == 0) { ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, @@ -114,11 +115,22 @@ void ConsoleReporter::PrintRunData(const Run& result) { (static_cast(result.iterations)), timeLabel); } + ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); - ColorPrintf(COLOR_DEFAULT, "%*s %*s %s\n", - 13, rate.c_str(), - 18, items.c_str(), - result.report_label.c_str()); + + if (!rate.empty()) { + ColorPrintf(COLOR_DEFAULT, " %*s", 13, rate.c_str()); + } + + if (!items.empty()) { + ColorPrintf(COLOR_DEFAULT, " %*s", 18, items.c_str()); + } + + if (!result.report_label.empty()) { + ColorPrintf(COLOR_DEFAULT, " %s", result.report_label.c_str()); + } + + ColorPrintf(COLOR_DEFAULT, "\n"); } } // end namespace benchmark -- cgit v1.2.3 From dce2ebb40394237f75e577e1a5e2b4dfc20eb006 Mon Sep 17 00:00:00 2001 From: Jussi Knuuttila Date: Sat, 30 Apr 2016 14:55:19 +0300 Subject: Fixed a warning caused by an implicit narrowing cast. --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 35b5e2d..3ee9c25 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -660,7 +660,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, thread.join(); } for (std::size_t ti = 0; ti < pool.size(); ++ti) { - pool[ti] = std::thread(&RunInThread, &b, iters, ti, &total); + pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti), &total); } } else { // Run directly in this thread -- cgit v1.2.3 From e253a284029c34f764fc26bb7859298d4ec0551b Mon Sep 17 00:00:00 2001 From: Jussi Knuuttila Date: Sat, 30 Apr 2016 16:23:58 +0300 Subject: Manual timing support. --- AUTHORS | 1 + CONTRIBUTORS | 1 + README.md | 39 ++++++++++++++++++++++++ include/benchmark/benchmark_api.h | 18 +++++++++++ src/benchmark.cc | 64 ++++++++++++++++++++++++++++++++++++--- test/benchmark_test.cc | 27 +++++++++++++++++ 6 files changed, 146 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5a4b355..9da43c7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -17,6 +17,7 @@ Evgeny Safronov Felix Homann Google Inc. JianXiong Zhou +Jussi Knuuttila Kaito Udagawa Lei Xu Matt Clarkson diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 2bbe19e..fcc3f35 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -31,6 +31,7 @@ Eugene Zhuk Evgeny Safronov Felix Homann JianXiong Zhou +Jussi Knuuttila Kaito Udagawa Kai Wolf Lei Xu diff --git a/README.md b/README.md index 2ccfd50..051b301 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,45 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. + +## Manual timing +For benchmarking something for which neither CPU time nor real-time are +correct or accurate enough, completely manual timing is supported using +the `UseManualTime` function. + +When `UseManualTime` is used, the benchmarked code must call +`SetIterationTime` once per iteration of the `KeepRunning` loop to +report the manually measured time. + +An example use case for this is benchmarking GPU execution (e.g. OpenCL +or CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot +be accurately measured using CPU time or real-time. Instead, they can be +measured accurately using a dedicated API, and these measurement results +can be reported back with `SetIterationTime`. + +```c++ +static void BM_ManualTiming(benchmark::State& state) { + int microseconds = state.range_x(); + std::chrono::duration sleep_duration { + static_cast(microseconds) + }; + + while (state.KeepRunning()) { + auto start = std::chrono::high_resolution_clock::now(); + // Simulate some useful workload with a sleep + std::this_thread::sleep_for(sleep_duration); + auto end = std::chrono::high_resolution_clock::now(); + + auto elapsed_seconds = + std::chrono::duration_cast>( + end - start); + + state.SetIterationTime(elapsed_seconds.count()); + } +} +BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime(); +``` + ### Preventing optimisation To prevent a value or expression from being optimized away by the compiler the `benchmark::DoNotOptimize(...)` function can be used. diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 726e35d..ef7eca5 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -283,6 +283,17 @@ public: // within each benchmark iteration, if possible. void ResumeTiming(); + // REQUIRES: called exactly once per iteration of the KeepRunning loop. + // Set the manually measured time for this benchmark iteration, which + // is used instead of automatically measured time if UseManualTime() was + // specified. + // + // For threaded benchmarks the SetIterationTime() function acts + // like a barrier. I.e., the ith call by a particular thread to this + // function will block until all threads have made their ith call. + // The time will be set by the last thread to call this function. + void SetIterationTime(double seconds); + // Set the number of bytes processed by the current benchmark // execution. This routine is typically called once at the end of a // throughput oriented benchmark. If this routine is called with a @@ -444,6 +455,13 @@ public: // called, the cpu time used by the benchmark will be used. Benchmark* UseRealTime(); + // If a benchmark must measure time manually (e.g. if GPU execution time is being + // measured), call this method. If called, each benchmark iteration should call + // SetIterationTime(seconds) to report the measured time, which will be used + // to control how many iterations are run, and in the printing of items/second + // or MB/second values. + Benchmark* UseManualTime(); + // Support for running multiple copies of the same benchmark concurrently // in multiple threads. This may be useful when measuring the scaling // of some piece of code. diff --git a/src/benchmark.cc b/src/benchmark.cc index 3ee9c25..1a836eb 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -130,6 +130,7 @@ class TimerManager { running_(false), real_time_used_(0), cpu_time_used_(0), + manual_time_used_(0), num_finalized_(0), phase_number_(0), entered_(0) { @@ -169,6 +170,21 @@ class TimerManager { } } + // Called by each thread + void SetIterationTime(double seconds) EXCLUDES(lock_) { + bool last_thread = false; + { + MutexLock ml(lock_); + last_thread = Barrier(ml); + if (last_thread) { + manual_time_used_ += seconds; + } + } + if (last_thread) { + phase_condition_.notify_all(); + } + } + // Called by each thread void Finalize() EXCLUDES(lock_) { MutexLock l(lock_); @@ -194,6 +210,13 @@ class TimerManager { return cpu_time_used_; } + // REQUIRES: timer is not running + double manual_time_used() EXCLUDES(lock_) { + MutexLock l(lock_); + CHECK(!running_); + return manual_time_used_; + } + private: Mutex lock_; Condition phase_condition_; @@ -207,6 +230,8 @@ class TimerManager { // Accumulated time so far (does not contain current slice if running_) double real_time_used_; double cpu_time_used_; + // Manually set iteration time. User sets this with SetIterationTime(seconds). + double manual_time_used_; // How many threads have called Finalize() int num_finalized_; @@ -263,6 +288,7 @@ struct Benchmark::Instance { int arg2; TimeUnit time_unit; bool use_real_time; + bool use_manual_time; double min_time; int threads; // Number of concurrent threads to use bool multithreaded; // Is benchmark multi-threaded? @@ -302,6 +328,7 @@ public: void RangePair(int lo1, int hi1, int lo2, int hi2); void MinTime(double n); void UseRealTime(); + void UseManualTime(); void Threads(int t); void ThreadRange(int min_threads, int max_threads); void ThreadPerCpu(); @@ -318,6 +345,7 @@ private: TimeUnit time_unit_; double min_time_; bool use_real_time_; + bool use_manual_time_; std::vector thread_counts_; BenchmarkImp& operator=(BenchmarkImp const&); @@ -378,6 +406,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.time_unit = family->time_unit_; instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; + instance.use_manual_time = family->use_manual_time_; instance.threads = num_threads; instance.multithreaded = !(family->thread_counts_.empty()); @@ -391,7 +420,9 @@ bool BenchmarkFamilies::FindBenchmarks( if (!IsZero(family->min_time_)) { instance.name += StringPrintF("/min_time:%0.3f", family->min_time_); } - if (family->use_real_time_) { + if (family->use_manual_time_) { + instance.name += "/manual_time"; + } else if (family->use_real_time_) { instance.name += "/real_time"; } @@ -411,7 +442,8 @@ bool BenchmarkFamilies::FindBenchmarks( BenchmarkImp::BenchmarkImp(const char* name) : name_(name), arg_count_(-1), time_unit_(kNanosecond), - min_time_(0.0), use_real_time_(false) { + min_time_(0.0), use_real_time_(false), + use_manual_time_(false) { } BenchmarkImp::~BenchmarkImp() { @@ -474,9 +506,15 @@ void BenchmarkImp::MinTime(double t) { } void BenchmarkImp::UseRealTime() { + CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; use_real_time_ = true; } +void BenchmarkImp::UseManualTime() { + CHECK(!use_real_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; + use_manual_time_ = true; +} + void BenchmarkImp::Threads(int t) { CHECK_GT(t, 0); thread_counts_.push_back(t); @@ -579,6 +617,11 @@ Benchmark* Benchmark::UseRealTime() { return this; } +Benchmark* Benchmark::UseManualTime() { + imp_->UseManualTime(); + return this; +} + Benchmark* Benchmark::Threads(int t) { imp_->Threads(t); return this; @@ -671,6 +714,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, const double cpu_accumulated_time = timer_manager->cpu_time_used(); const double real_accumulated_time = timer_manager->real_time_used(); + const double manual_accumulated_time = timer_manager->manual_time_used(); timer_manager.reset(); VLOG(2) << "Ran in " << cpu_accumulated_time << "/" @@ -678,7 +722,9 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, // Base decisions off of real time if requested by this benchmark. double seconds = cpu_accumulated_time; - if (b.use_real_time) { + if (b.use_manual_time) { + seconds = manual_accumulated_time; + } else if (b.use_real_time) { seconds = real_accumulated_time; } @@ -713,7 +759,11 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, // Report the total iterations across all threads. report.iterations = static_cast(iters) * b.threads; report.time_unit = b.time_unit; - report.real_accumulated_time = real_accumulated_time; + if (b.use_manual_time) { + report.real_accumulated_time = manual_accumulated_time; + } else { + report.real_accumulated_time = real_accumulated_time; + } report.cpu_accumulated_time = cpu_accumulated_time; report.bytes_per_second = bytes_per_second; report.items_per_second = items_per_second; @@ -774,6 +824,12 @@ void State::ResumeTiming() { timer_manager->StartTimer(); } +void State::SetIterationTime(double seconds) +{ + CHECK(running_benchmark); + timer_manager->SetIterationTime(seconds); +} + void State::SetLabel(const char* label) { CHECK(running_benchmark); MutexLock l(GetBenchmarkLock()); diff --git a/test/benchmark_test.cc b/test/benchmark_test.cc index 97abb68..fa99559 100644 --- a/test/benchmark_test.cc +++ b/test/benchmark_test.cc @@ -14,6 +14,8 @@ #include #include #include +#include +#include #if defined(__GNUC__) # define BENCHMARK_NOINLINE __attribute__((noinline)) @@ -174,5 +176,30 @@ static void BM_ParallelMemset(benchmark::State& state) { } BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4); +static void BM_ManualTiming(benchmark::State& state) { + size_t slept_for = 0; + int microseconds = state.range_x(); + std::chrono::duration sleep_duration { + static_cast(microseconds) + }; + + while (state.KeepRunning()) { + auto start = std::chrono::high_resolution_clock::now(); + // Simulate some useful workload with a sleep + std::this_thread::sleep_for(sleep_duration); + auto end = std::chrono::high_resolution_clock::now(); + + auto elapsed = + std::chrono::duration_cast>( + end - start); + + state.SetIterationTime(elapsed.count()); + slept_for += microseconds; + } + state.SetItemsProcessed(slept_for); +} +BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime(); +BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime(); + BENCHMARK_MAIN() -- cgit v1.2.3 From fa8b2d617152ee840ee924910cf3f20bf9c61277 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 10 May 2016 16:46:26 -0700 Subject: Add myself to the contributors list --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index fcc3f35..67ecb28 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -23,6 +23,7 @@ # Please keep the list sorted. Arne Beer +Billy Robert O'Neal III Chris Kennelly Christopher Seymour David Coeurjolly -- cgit v1.2.3 From 60b59217359ecac2060637906e9eae14216a47d7 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 27 Apr 2016 18:44:39 -0700 Subject: Remove redundant MSVC++ /W3 triggering warnings This change looks for CMake's default setting for MSVC++, /W3 (and any other level should that change in the future), and removes it before adding /W4. This stops the build for MSVC++ emitting warnings about /W4 overriding /W3 earlier on the command line. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c72252..5940561 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,8 @@ include(CXXFeatureCheck) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # Turn compiler warnings up to 11 - add_cxx_compiler_flag(-W4) + string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") add_definitions(-D_CRT_SECURE_NO_WARNINGS) # Link time optimisation -- cgit v1.2.3 From 9bb1ba6b76f6af2f32e2f146c3e97fdac67e8340 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 27 Apr 2016 18:47:03 -0700 Subject: Fix clobbering of default CMAKE_CXX_FLAGS_RELEASE BENCHMARK_ENABLE_LTO=true was completely replacing CMAKE_CXX_FLAGS_RELEASE; meaning neither CMake's release defaults nor user customizations were being applied. --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5940561..4424527 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,10 +39,10 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # Link time optimisation if (BENCHMARK_ENABLE_LTO) - set(CMAKE_CXX_FLAGS_RELEASE "/GL") - set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "/LTCG") - set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "/LTCG") - set(CMAKE_EXE_LINKER_FLAGS_RELEASE "/LTCG") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GL") + set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "${CMAKE_STATIC_LINKER_FLAGS_RELEASE} /LTCG") + set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG") endif() else() # Try and enable C++11. Don't use C++14 because it doesn't work in some -- cgit v1.2.3 From d89ab075fd0b545f4c69732839e2a978df602abf Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 27 Apr 2016 18:48:43 -0700 Subject: Added support for CMake's other rel configs In addition to release, CMake supports RELWITHDEBINFO and MINSIZEREL build configurations. In particular, debug info is necessary for many profilers to do anything useful, making RELWITHDEBINFO important here. MINSIZEREL was added for completeness' sake. --- CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4424527..708fc7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,19 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "${CMAKE_STATIC_LINKER_FLAGS_RELEASE} /LTCG") set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG") + + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /GL") + string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO}") + set(CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO} /LTCG") + string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO}") + set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /LTCG") + string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}") + set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /LTCG") + + set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /GL") + set(CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL "${CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL} /LTCG") + set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL "${CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL} /LTCG") + set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG") endif() else() # Try and enable C++11. Don't use C++14 because it doesn't work in some -- cgit v1.2.3 From 09edc486b851ab948cc859e2c4a85a189378b284 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 27 Apr 2016 18:49:17 -0700 Subject: Added GCC/Clang support for other rel configs --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 708fc7f..f340fb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,8 @@ else() add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wshadow) add_cxx_compiler_flag(-Werror RELEASE) + add_cxx_compiler_flag(-Werror RELWITHDEBINFO) + add_cxx_compiler_flag(-Werror MINSIZEREL) add_cxx_compiler_flag(-pedantic) add_cxx_compiler_flag(-pedantic-errors) add_cxx_compiler_flag(-Wshorten-64-to-32) -- cgit v1.2.3 From df9ab80113a890c38ff93ef37699078eeceb29fc Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 10 May 2016 17:35:36 -0700 Subject: Use nanoseconds instead of duration MSVC++ before 2015 Update 2 has a bug in sleep_for where it tries to implicitly += the input with a nanoseconds variable. Work around this by using nanoseconds directly (which can be implicitly +='d with chrono::nanoseconds). --- test/benchmark_test.cc | 3 ++- test/options_test.cc | 11 ++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/test/benchmark_test.cc b/test/benchmark_test.cc index fa99559..252602a 100644 --- a/test/benchmark_test.cc +++ b/test/benchmark_test.cc @@ -186,7 +186,8 @@ static void BM_ManualTiming(benchmark::State& state) { while (state.KeepRunning()) { auto start = std::chrono::high_resolution_clock::now(); // Simulate some useful workload with a sleep - std::this_thread::sleep_for(sleep_duration); + std::this_thread::sleep_for(std::chrono::duration_cast< + std::chrono::nanoseconds>(sleep_duration)); auto end = std::chrono::high_resolution_clock::now(); auto elapsed = diff --git a/test/options_test.cc b/test/options_test.cc index 4fe2d17..4737caa 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -9,14 +9,11 @@ void BM_basic(benchmark::State& state) { } void BM_basic_slow(benchmark::State& state) { - - int milliseconds = state.range_x(); - std::chrono::duration sleep_duration { - static_cast(milliseconds) - }; - + std::chrono::milliseconds sleep_duration(state.range_x()); while (state.KeepRunning()) { - std::this_thread::sleep_for(sleep_duration); + std::this_thread::sleep_for( + std::chrono::duration_cast(sleep_duration) + ); } } -- cgit v1.2.3 From fa0e7ef8c66f85007170e69f28afe9e1b86a957f Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 11 May 2016 11:16:48 -0700 Subject: Mark sudo: required for Travis Google Benchmark's Travis build currently requires "sudo" to install newer versions of CMake and similar. See this for more details: https://docs.travis-ci.com/user/workers/container-based-infrastructure/ Since Google Benchmark was put into Travis before 2015-01-01, it gets the standard infrastructure implicitly, so sudo works. But anyone who forks this repository and tries to add Travis.CI (so they can see if the build works before creating a PR) gets broken builds before this change. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8b138ce..bf26395 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,3 +39,5 @@ after_success: - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then coveralls --include src --include include --gcov-options '\-lp' --root .. --build-root .; fi + +sudo: required -- cgit v1.2.3 From c60eefdbb78b29522568b1055941f3c20c843e78 Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 14 May 2016 15:56:34 +0200 Subject: added option to change range multiplier --- include/benchmark/benchmark_api.h | 4 ++++ src/benchmark.cc | 24 +++++++++++++++++++----- test/options_test.cc | 1 + 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index ef7eca5..2ded481 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -444,6 +444,10 @@ public: // Threads, etc. Benchmark* Apply(void (*func)(Benchmark* benchmark)); + // Set the range multiplier for non-dense range. If not called, the range multiplier + // kRangeMultiplier will be used. + Benchmark* RangeMultiplier(int multiplier); + // Set the minimum amount of time to use when running this benchmark. This // option overrides the `benchmark_min_time` flag. Benchmark* MinTime(double t); diff --git a/src/benchmark.cc b/src/benchmark.cc index 1a836eb..153cb95 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -287,6 +287,7 @@ struct Benchmark::Instance { bool has_arg2; int arg2; TimeUnit time_unit; + int range_multiplier; bool use_real_time; bool use_manual_time; double min_time; @@ -326,6 +327,7 @@ public: void DenseRange(int start, int limit); void ArgPair(int start, int limit); void RangePair(int lo1, int hi1, int lo2, int hi2); + void RangeMultiplier(int multiplier); void MinTime(double n); void UseRealTime(); void UseManualTime(); @@ -343,6 +345,7 @@ private: int arg_count_; std::vector< std::pair > args_; // Args for all benchmark runs TimeUnit time_unit_; + int range_multiplier_; double min_time_; bool use_real_time_; bool use_manual_time_; @@ -404,6 +407,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.has_arg2 = family->arg_count_ == 2; instance.arg2 = args.second; instance.time_unit = family->time_unit_; + instance.range_multiplier = family->range_multiplier_; instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; @@ -442,8 +446,8 @@ bool BenchmarkFamilies::FindBenchmarks( BenchmarkImp::BenchmarkImp(const char* name) : name_(name), arg_count_(-1), time_unit_(kNanosecond), - min_time_(0.0), use_real_time_(false), - use_manual_time_(false) { + range_multiplier_(kRangeMultiplier), min_time_(0.0), + use_real_time_(false), use_manual_time_(false) { } BenchmarkImp::~BenchmarkImp() { @@ -463,7 +467,7 @@ void BenchmarkImp::Range(int start, int limit) { CHECK(arg_count_ == -1 || arg_count_ == 1); arg_count_ = 1; std::vector arglist; - AddRange(&arglist, start, limit, kRangeMultiplier); + AddRange(&arglist, start, limit, range_multiplier_); for (int i : arglist) { args_.emplace_back(i, -1); @@ -490,8 +494,8 @@ void BenchmarkImp::RangePair(int lo1, int hi1, int lo2, int hi2) { CHECK(arg_count_ == -1 || arg_count_ == 2); arg_count_ = 2; std::vector arglist1, arglist2; - AddRange(&arglist1, lo1, hi1, kRangeMultiplier); - AddRange(&arglist2, lo2, hi2, kRangeMultiplier); + AddRange(&arglist1, lo1, hi1, range_multiplier_); + AddRange(&arglist2, lo2, hi2, range_multiplier_); for (int i : arglist1) { for (int j : arglist2) { @@ -500,6 +504,11 @@ void BenchmarkImp::RangePair(int lo1, int hi1, int lo2, int hi2) { } } +void BenchmarkImp::RangeMultiplier(int multiplier) { + CHECK_GE(multiplier, 2); + range_multiplier_ = multiplier; +} + void BenchmarkImp::MinTime(double t) { CHECK(t > 0.0); min_time_ = t; @@ -607,6 +616,11 @@ Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) { return this; } +Benchmark* Benchmark::RangeMultiplier(int multiplier) { + imp_->RangeMultiplier(multiplier); + return this; +} + Benchmark* Benchmark::MinTime(double t) { imp_->MinTime(t); return this; diff --git a/test/options_test.cc b/test/options_test.cc index 4737caa..ed5fa68 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -23,6 +23,7 @@ BENCHMARK(BM_basic_slow)->Arg(10)->Unit(benchmark::kNanosecond); BENCHMARK(BM_basic_slow)->Arg(100)->Unit(benchmark::kMicrosecond); BENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kMillisecond); BENCHMARK(BM_basic)->Range(1, 8); +BENCHMARK(BM_basic)->RangeMultiplier(2)->Range(1, 8); BENCHMARK(BM_basic)->DenseRange(10, 15); BENCHMARK(BM_basic)->ArgPair(42, 42); BENCHMARK(BM_basic)->RangePair(64, 512, 64, 512); -- cgit v1.2.3 From d2103de3d390715845f0cb42d3f070a039814580 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 16 May 2016 18:17:11 +0200 Subject: Modified check for range multiplier --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 153cb95..3d12d28 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -505,7 +505,6 @@ void BenchmarkImp::RangePair(int lo1, int hi1, int lo2, int hi2) { } void BenchmarkImp::RangeMultiplier(int multiplier) { - CHECK_GE(multiplier, 2); range_multiplier_ = multiplier; } @@ -548,6 +547,7 @@ void BenchmarkImp::SetName(const char* name) { void BenchmarkImp::AddRange(std::vector* dst, int lo, int hi, int mult) { CHECK_GE(lo, 0); CHECK_GE(hi, lo); + CHECK_GE(mult, 2); // Add "lo" dst->push_back(lo); -- cgit v1.2.3 From 27f0baa190b4ebd31436b3e8a89bee7fbdc50eb9 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 18 May 2016 19:59:34 +0200 Subject: Added test file and Complexity() interface --- include/benchmark/benchmark_api.h | 18 +++++++ src/benchmark.cc | 16 +++++- test/CMakeLists.txt | 5 +- test/complexity_test.cc | 107 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 test/complexity_test.cc diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 2ded481..8878b58 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -231,6 +231,20 @@ enum TimeUnit { kMillisecond }; +// BigO is passed to a benchmark in order to specify the asymptotic computational +// complexity for the benchmark. +enum BigO { + O_None, + O_1, + O_N, + O_M_plus_N, + O_N_Squared, + O_N_Cubed, + O_log_N, + O_N_log_N, + O_Auto +}; + // State is passed to a running Benchmark and contains state for the // benchmark to use. class State { @@ -465,6 +479,10 @@ public: // to control how many iterations are run, and in the printing of items/second // or MB/second values. Benchmark* UseManualTime(); + + // Set the asymptotic computational complexity for the benchmark. This option + // called the asymptotic computational complexity will be shown on the output. + Benchmark* Complexity(BigO complexity); // Support for running multiple copies of the same benchmark concurrently // in multiple threads. This may be useful when measuring the scaling diff --git a/src/benchmark.cc b/src/benchmark.cc index 3d12d28..afe1e44 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -290,6 +290,7 @@ struct Benchmark::Instance { int range_multiplier; bool use_real_time; bool use_manual_time; + BigO complexity; double min_time; int threads; // Number of concurrent threads to use bool multithreaded; // Is benchmark multi-threaded? @@ -331,6 +332,7 @@ public: void MinTime(double n); void UseRealTime(); void UseManualTime(); + void Complexity(BigO complexity); void Threads(int t); void ThreadRange(int min_threads, int max_threads); void ThreadPerCpu(); @@ -349,6 +351,7 @@ private: double min_time_; bool use_real_time_; bool use_manual_time_; + BigO complexity_; std::vector thread_counts_; BenchmarkImp& operator=(BenchmarkImp const&); @@ -411,6 +414,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; + instance.complexity = family->complexity_; instance.threads = num_threads; instance.multithreaded = !(family->thread_counts_.empty()); @@ -447,7 +451,8 @@ bool BenchmarkFamilies::FindBenchmarks( BenchmarkImp::BenchmarkImp(const char* name) : name_(name), arg_count_(-1), time_unit_(kNanosecond), range_multiplier_(kRangeMultiplier), min_time_(0.0), - use_real_time_(false), use_manual_time_(false) { + use_real_time_(false), use_manual_time_(false), + complexity_(O_None) { } BenchmarkImp::~BenchmarkImp() { @@ -523,6 +528,10 @@ void BenchmarkImp::UseManualTime() { use_manual_time_ = true; } +void BenchmarkImp::Complexity(BigO complexity){ + complexity_ = complexity; +} + void BenchmarkImp::Threads(int t) { CHECK_GT(t, 0); thread_counts_.push_back(t); @@ -636,6 +645,11 @@ Benchmark* Benchmark::UseManualTime() { return this; } +Benchmark* Benchmark::Complexity(BigO complexity) { + imp_->Complexity(complexity); + return this; +} + Benchmark* Benchmark::Threads(int t) { imp_->Threads(t); return this; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a10a53a..1bc9dfe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -47,6 +47,9 @@ set_target_properties(cxx03_test PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}") add_test(cxx03 cxx03_test --benchmark_min_time=0.01) +compile_benchmark_test(complexity_test) +add_test(complexity_benchmark complexity_test --benchmark_min_time=0.01) + # Add the coverage command(s) if(CMAKE_BUILD_TYPE) string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) @@ -66,7 +69,7 @@ if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage") COMMAND ${LCOV} -q -a before.lcov -a after.lcov --output-file final.lcov COMMAND ${LCOV} -q -r final.lcov "'${CMAKE_SOURCE_DIR}/test/*'" -o final.lcov COMMAND ${GENHTML} final.lcov -o lcov --demangle-cpp --sort -p "${CMAKE_BINARY_DIR}" -t benchmark - DEPENDS filter_test benchmark_test options_test basic_test fixture_test cxx03_test + DEPENDS filter_test benchmark_test options_test basic_test fixture_test cxx03_test complexity_test WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Running LCOV" ) diff --git a/test/complexity_test.cc b/test/complexity_test.cc new file mode 100644 index 0000000..777c0f3 --- /dev/null +++ b/test/complexity_test.cc @@ -0,0 +1,107 @@ + +#include "benchmark/benchmark_api.h" + +#include +#include +#include +#include + +std::vector ConstructRandomVector(int size) { + std::vector v; + v.reserve(size); + for (int i = 0; i < size; ++i) { + v.push_back(rand() % size); + } + return v; +} + +std::map ConstructRandomMap(int size) { + std::map m; + for (int i = 0; i < size; ++i) { + m.insert(std::make_pair(rand() % size, rand() % size)); + } + return m; +} + +void BM_Complexity_O1(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<17) -> Complexity(benchmark::O_1); + +static void BM_Complexity_O_N(benchmark::State& state) { + auto v = ConstructRandomVector(state.range_x()); + const int itemNotInVector = state.range_x()*2; // Test worst case scenario (item not in vector) + while (state.KeepRunning()) { + benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector)); + } +} +BENCHMARK(BM_Complexity_O_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_N); +BENCHMARK(BM_Complexity_O_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_Auto); + +static void BM_Complexity_O_M_plus_N(benchmark::State& state) { + std::string s1(state.range_x(), '-'); + std::string s2(state.range_x(), '-'); + while (state.KeepRunning()) + benchmark::DoNotOptimize(s1.compare(s2)); +} +BENCHMARK(BM_Complexity_O_M_plus_N) + ->RangeMultiplier(2)->Range(1<<10, 1<<18) -> Complexity(benchmark::O_M_plus_N); + +static void BM_Complexity_O_N_Squared(benchmark::State& state) { + std::string s1(state.range_x(), '-'); + std::string s2(state.range_x(), '-'); + while (state.KeepRunning()) + for(char& c1 : s1) { + for(char& c2 : s2) { + benchmark::DoNotOptimize(c1 = 'a'); + benchmark::DoNotOptimize(c2 = 'b'); + } + } +} +BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::O_N_Squared); + +static void BM_Complexity_O_N_Cubed(benchmark::State& state) { + std::string s1(state.range_x(), '-'); + std::string s2(state.range_x(), '-'); + std::string s3(state.range_x(), '-'); + while (state.KeepRunning()) + for(char& c1 : s1) { + for(char& c2 : s2) { + for(char& c3 : s3) { + benchmark::DoNotOptimize(c1 = 'a'); + benchmark::DoNotOptimize(c2 = 'b'); + benchmark::DoNotOptimize(c3 = 'c'); + } + } + } +} +BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::O_N_Cubed); + +static void BM_Complexity_O_log_N(benchmark::State& state) { + auto m = ConstructRandomMap(state.range_x()); + const int itemNotInVector = state.range_x()*2; // Test worst case scenario (item not in vector) + while (state.KeepRunning()) { + benchmark::DoNotOptimize(m.find(itemNotInVector)); + } +} +BENCHMARK(BM_Complexity_O_log_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_log_N); + +static void BM_Complexity_O_N_log_N(benchmark::State& state) { + auto v = ConstructRandomVector(state.range_x()); + while (state.KeepRunning()) { + std::sort(v.begin(), v.end()); + } +} +BENCHMARK(BM_Complexity_O_N_log_N) -> Range(1, 1<<16) -> Complexity(benchmark::O_N_log_N); +BENCHMARK(BM_Complexity_O_N_log_N) -> Range(1, 1<<16) -> Complexity(benchmark::O_Auto); + +// Test benchmark with no range. Complexity is always calculated as O(1). +void BM_Extreme_Cases(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_Extreme_Cases); +BENCHMARK(BM_Extreme_Cases)->Arg(42); + +BENCHMARK_MAIN() \ No newline at end of file -- cgit v1.2.3 From b73dc22944cb933289bbdbf5bb6616dbfc50168f Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 18 May 2016 21:25:00 +0200 Subject: implemented Complexity for O(1) --- include/benchmark/reporter.h | 22 ++++++++++++-- src/benchmark.cc | 23 +++++++++++++-- src/console_reporter.cc | 17 ++++++++++- src/csv_reporter.cc | 21 ++++++++++++-- src/json_reporter.cc | 17 ++++++++++- src/reporter.cc | 68 +++++++++++++++++++++++++++++++++----------- test/complexity_test.cc | 4 +-- 7 files changed, 145 insertions(+), 27 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index aaf5fbf..b398800 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -48,7 +48,10 @@ class BenchmarkReporter { cpu_accumulated_time(0), bytes_per_second(0), items_per_second(0), - max_heapbytes_used(0) {} + max_heapbytes_used(0), + complexity(O_1), + arg1(0), + arg2(0) {} std::string benchmark_name; std::string report_label; // Empty if not set by benchmark. @@ -63,6 +66,11 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; + + // Keep track of arguments to compute asymptotic complexity + BigO complexity; + int arg1; + int arg2; }; // Called once for every suite of benchmarks run. @@ -78,6 +86,12 @@ class BenchmarkReporter { // Note that all the grouped benchmark runs should refer to the same // benchmark, thus have the same name. virtual void ReportRuns(const std::vector& report) = 0; + + // Called once at the last instance of a benchmark range, gives information about + // asymptotic complexity and RMS. + // Note that all the benchmark runs in a range should refer to the same benchmark, + // thus have the same name. + virtual void ReportComplexity(const std::vector& complexity_reports) = 0; // Called once and only once after ever group of benchmarks is run and // reported. @@ -85,7 +99,8 @@ class BenchmarkReporter { virtual ~BenchmarkReporter(); protected: - static void ComputeStats(std::vector const& reports, Run* mean, Run* stddev); + static void ComputeStats(const std::vector & reports, Run& mean, Run& stddev); + static void ComputeBigO(const std::vector & reports, Run& bigO, Run& rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); }; @@ -95,6 +110,7 @@ class ConsoleReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); + virtual void ReportComplexity(const std::vector& complexity_reports); protected: virtual void PrintRunData(const Run& report); @@ -107,6 +123,7 @@ public: JSONReporter() : first_report_(true) {} virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); + virtual void ReportComplexity(const std::vector& complexity_reports); virtual void Finalize(); private: @@ -119,6 +136,7 @@ class CSVReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); + virtual void ReportComplexity(const std::vector& complexity_reports); private: void PrintRunData(const Run& report); diff --git a/src/benchmark.cc b/src/benchmark.cc index afe1e44..874dc0c 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -291,6 +291,7 @@ struct Benchmark::Instance { bool use_real_time; bool use_manual_time; BigO complexity; + bool last_benchmark_instance; double min_time; int threads; // Number of concurrent threads to use bool multithreaded; // Is benchmark multi-threaded? @@ -414,6 +415,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; + instance.last_benchmark_instance = (args == family->args_.back()); instance.complexity = family->complexity_; instance.threads = num_threads; instance.multithreaded = !(family->thread_counts_.empty()); @@ -697,7 +699,8 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, } void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, - BenchmarkReporter* br) EXCLUDES(GetBenchmarkLock()) { + BenchmarkReporter* br, + std::vector& complexity_reports) EXCLUDES(GetBenchmarkLock()) { size_t iters = 1; std::vector reports; @@ -795,7 +798,14 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.cpu_accumulated_time = cpu_accumulated_time; report.bytes_per_second = bytes_per_second; report.items_per_second = items_per_second; + report.arg1 = b.arg1; + report.arg2 = b.arg2; + report.complexity = b.complexity; reports.push_back(report); + + if(report.complexity != O_None) + complexity_reports.push_back(report); + break; } @@ -819,6 +829,12 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, } } br->ReportRuns(reports); + + if((b.complexity != O_None) && b.last_benchmark_instance) { + br->ReportComplexity(complexity_reports); + complexity_reports.clear(); + } + if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); @@ -903,9 +919,12 @@ void RunMatchingBenchmarks(const std::string& spec, context.cpu_scaling_enabled = CpuScalingEnabled(); context.name_field_width = name_field_width; + // Keep track of runing times of all instances of current benchmark + std::vector complexity_reports; + if (reporter->ReportContext(context)) { for (const auto& benchmark : benchmarks) { - RunBenchmark(benchmark, reporter); + RunBenchmark(benchmark, reporter, complexity_reports); } } } diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 56bd3ce..0d8ab1d 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -72,13 +72,28 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); // Output using PrintRun. PrintRunData(mean_data); PrintRunData(stddev_data); } +void ConsoleReporter::ReportComplexity(const std::vector & complexity_reports) { + if (complexity_reports.size() < 2) { + // We don't report asymptotic complexity data if there was a single run. + return; + } + + Run bigO_data; + Run rms_data; + BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + + // Output using PrintRun. + PrintRunData(bigO_data); + PrintRunData(rms_data); +} + void ConsoleReporter::PrintRunData(const Run& result) { // Format bytes per second std::string rate; diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 3f67d1d..f13a5f8 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -48,7 +48,7 @@ bool CSVReporter::ReportContext(const Context& context) { return true; } -void CSVReporter::ReportRuns(std::vector const& reports) { +void CSVReporter::ReportRuns(const std::vector & reports) { if (reports.empty()) { return; } @@ -57,7 +57,7 @@ void CSVReporter::ReportRuns(std::vector const& reports) { if (reports.size() >= 2) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); reports_cp.push_back(mean_data); reports_cp.push_back(stddev_data); } @@ -66,7 +66,22 @@ void CSVReporter::ReportRuns(std::vector const& reports) { } } -void CSVReporter::PrintRunData(Run const& run) { +void CSVReporter::ReportComplexity(const std::vector & complexity_reports) { + if (complexity_reports.size() < 2) { + // We don't report asymptotic complexity data if there was a single run. + return; + } + + Run bigO_data; + Run rms_data; + BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + + // Output using PrintRun. + PrintRunData(bigO_data); + PrintRunData(rms_data); +} + +void CSVReporter::PrintRunData(const Run & run) { double multiplier; const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 7ed141f..07fc366 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -100,7 +100,7 @@ void JSONReporter::ReportRuns(std::vector const& reports) { if (reports.size() >= 2) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); reports_cp.push_back(mean_data); reports_cp.push_back(stddev_data); } @@ -115,6 +115,21 @@ void JSONReporter::ReportRuns(std::vector const& reports) { } } +void JSONReporter::ReportComplexity(const std::vector & complexity_reports) { + if (complexity_reports.size() < 2) { + // We don't report asymptotic complexity data if there was a single run. + return; + } + + Run bigO_data; + Run rms_data; + BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + + // Output using PrintRun. + PrintRunData(bigO_data); + PrintRunData(rms_data); +} + void JSONReporter::Finalize() { // Close the list of benchmarks and the top level object. std::cout << "\n ]\n}\n"; diff --git a/src/reporter.cc b/src/reporter.cc index 036546e..fd97aba 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -24,7 +24,7 @@ namespace benchmark { void BenchmarkReporter::ComputeStats( const std::vector& reports, - Run* mean_data, Run* stddev_data) { + Run& mean_data, Run& stddev_data) { CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports"; // Accumulators. Stat1_d real_accumulated_time_stat; @@ -48,33 +48,69 @@ void BenchmarkReporter::ComputeStats( } // Get the data from the accumulator to BenchmarkReporter::Run's. - mean_data->benchmark_name = reports[0].benchmark_name + "_mean"; - mean_data->iterations = run_iterations; - mean_data->real_accumulated_time = real_accumulated_time_stat.Mean() * + mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; + mean_data.iterations = run_iterations; + mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * run_iterations; - mean_data->cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * + mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * run_iterations; - mean_data->bytes_per_second = bytes_per_second_stat.Mean(); - mean_data->items_per_second = items_per_second_stat.Mean(); + mean_data.bytes_per_second = bytes_per_second_stat.Mean(); + mean_data.items_per_second = items_per_second_stat.Mean(); // Only add label to mean/stddev if it is same for all runs - mean_data->report_label = reports[0].report_label; + mean_data.report_label = reports[0].report_label; for (std::size_t i = 1; i < reports.size(); i++) { if (reports[i].report_label != reports[0].report_label) { - mean_data->report_label = ""; + mean_data.report_label = ""; break; } } - stddev_data->benchmark_name = reports[0].benchmark_name + "_stddev"; - stddev_data->report_label = mean_data->report_label; - stddev_data->iterations = 0; - stddev_data->real_accumulated_time = + stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; + stddev_data.report_label = mean_data.report_label; + stddev_data.iterations = 0; + stddev_data.real_accumulated_time = real_accumulated_time_stat.StdDev(); - stddev_data->cpu_accumulated_time = + stddev_data.cpu_accumulated_time = + cpu_accumulated_time_stat.StdDev(); + stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); + stddev_data.items_per_second = items_per_second_stat.StdDev(); +} + +void BenchmarkReporter::ComputeBigO( + const std::vector& reports, + Run& bigO, Run& rms) { + CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; + // Accumulators. + Stat1_d real_accumulated_time_stat; + Stat1_d cpu_accumulated_time_stat; + + // Populate the accumulators. + for (Run const& run : reports) { + real_accumulated_time_stat += + Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); + cpu_accumulated_time_stat += + Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); + } + + std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); + + // Get the data from the accumulator to BenchmarkReporter::Run's. + bigO.benchmark_name = benchmark_name + "_BigO"; + bigO.iterations = 0; + bigO.real_accumulated_time = real_accumulated_time_stat.Mean(); + bigO.cpu_accumulated_time = cpu_accumulated_time_stat.Mean(); + + // Only add label to mean/stddev if it is same for all runs + bigO.report_label = reports[0].report_label; + + rms.benchmark_name = benchmark_name + "_RMS"; + rms.report_label = bigO.report_label; + rms.iterations = 0; + rms.real_accumulated_time = + real_accumulated_time_stat.StdDev(); + rms.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); - stddev_data->bytes_per_second = bytes_per_second_stat.StdDev(); - stddev_data->items_per_second = items_per_second_stat.StdDev(); } TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) { diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 777c0f3..afa82ed 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -101,7 +101,7 @@ void BM_Extreme_Cases(benchmark::State& state) { while (state.KeepRunning()) { } } -BENCHMARK(BM_Extreme_Cases); -BENCHMARK(BM_Extreme_Cases)->Arg(42); +BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N); +BENCHMARK(BM_Extreme_Cases)->Arg(42) -> Complexity(benchmark::O_Auto); BENCHMARK_MAIN() \ No newline at end of file -- cgit v1.2.3 From 872ff01a49390ccaf8ee5f13c18ae7be9cce8275 Mon Sep 17 00:00:00 2001 From: Ismael Date: Fri, 20 May 2016 16:49:39 +0200 Subject: addaptation of minimal_leastsq library --- include/benchmark/benchmark_api.h | 17 +++--- src/CMakeLists.txt | 2 +- src/minimal_leastsq.cc | 113 ++++++++++++++++++++++++++++++++++++++ src/minimal_leastsq.h | 46 ++++++++++++++++ src/reporter.cc | 1 + test/complexity_test.cc | 9 --- 6 files changed, 169 insertions(+), 19 deletions(-) create mode 100644 src/minimal_leastsq.cc create mode 100644 src/minimal_leastsq.h diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 8878b58..146a8cc 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -234,15 +234,14 @@ enum TimeUnit { // BigO is passed to a benchmark in order to specify the asymptotic computational // complexity for the benchmark. enum BigO { - O_None, - O_1, - O_N, - O_M_plus_N, - O_N_Squared, - O_N_Cubed, - O_log_N, - O_N_log_N, - O_Auto + O_None, + O_1, + O_N, + O_N_Squared, + O_N_Cubed, + O_log_N, + O_N_log_N, + O_Auto }; // State is passed to a running Benchmark and contains state for the diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 811d075..a681b35 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories(${PROJECT_SOURCE_DIR}/src) set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc" "console_reporter.cc" "csv_reporter.cc" "json_reporter.cc" "log.cc" "reporter.cc" "sleep.cc" "string_util.cc" - "sysinfo.cc" "walltime.cc") + "sysinfo.cc" "walltime.cc" "minimal_leastsq.cc") # Determine the correct regular expression engine to use if(HAVE_STD_REGEX) set(RE_FILES "re_std.cc") diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc new file mode 100644 index 0000000..c4627d3 --- /dev/null +++ b/src/minimal_leastsq.cc @@ -0,0 +1,113 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Addapted to be used with google benchmark + +#include "minimal_leastsq.h" + +#include + +// Internal function to calculate the different scalability forms +double fittingCurve(double N, benchmark::BigO Complexity) { + if (Complexity == benchmark::O_N) + return N; + else if (Complexity == benchmark::O_N_Squared) + return pow(N, 2); + else if (Complexity == benchmark::O_N_Cubed) + return pow(N, 3); + else if (Complexity == benchmark::O_log_N) + return log2(N); + else if (Complexity == benchmark::O_N_log_N) + return N * log2(N); + + return 1; // Default value for O_1 +} + +// Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. +// - N : Vector containing the size of the benchmark tests. +// - Time : Vector containing the times for the benchmark tests. +// - Complexity : Fitting curve. +// For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit + +LeastSq leastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { + assert(N.size() == Time.size() && N.size() >= 2); + assert(Complexity != benchmark::O_None && + Complexity != benchmark::O_Auto); + + double sigmaGN = 0; + double sigmaGNSquared = 0; + double sigmaTime = 0; + double sigmaTimeGN = 0; + + // Calculate least square fitting parameter + for (size_t i = 0; i < N.size(); ++i) { + double GNi = fittingCurve(N[i], Complexity); + sigmaGN += GNi; + sigmaGNSquared += GNi * GNi; + sigmaTime += Time[i]; + sigmaTimeGN += Time[i] * GNi; + } + + LeastSq result; + result.complexity = Complexity; + + // Calculate complexity. + // O_1 is treated as an special case + if (Complexity != benchmark::O_1) + result.coef = sigmaTimeGN / sigmaGNSquared; + else + result.coef = sigmaTime / N.size(); + + // Calculate RMS + double rms = 0; + for (size_t i = 0; i < N.size(); ++i) { + double fit = result.coef * fittingCurve(N[i], Complexity); + rms += pow((Time[i] - fit), 2); + } + + double mean = sigmaTime / N.size(); + + result.rms = sqrt(rms) / mean; // Normalized RMS by the mean of the observed values + + return result; +} + +// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. +// - N : Vector containing the size of the benchmark tests. +// - Time : Vector containing the times for the benchmark tests. +// - Complexity : If different than O_Auto, the fitting curve will stick to this one. If it is O_Auto, it will be calculated +// the best fitting curve. + +LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { + assert(N.size() == Time.size() && N.size() >= 2); // Do not compute fitting curve is less than two benchmark runs are given + assert(Complexity != benchmark::O_None); // Check that complexity is a valid parameter. + + if(Complexity == benchmark::O_Auto) { + std::vector fitCurves = { benchmark::O_log_N, benchmark::O_N, benchmark::O_N_log_N, benchmark::O_N_Squared, benchmark::O_N_Cubed }; + + LeastSq best_fit = leastSq(N, Time, benchmark::O_1); // Take O_1 as default best fitting curve + + // Compute all possible fitting curves and stick to the best one + for (const auto& fit : fitCurves) { + LeastSq current_fit = leastSq(N, Time, fit); + if (current_fit.rms < best_fit.rms) + best_fit = current_fit; + } + + return best_fit; + } + else + return leastSq(N, Time, Complexity); +} \ No newline at end of file diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h new file mode 100644 index 0000000..ae725d1 --- /dev/null +++ b/src/minimal_leastsq.h @@ -0,0 +1,46 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Addapted to be used with google benchmark + +#if !defined(MINIMAL_LEASTSQ_H_) +#define MINIMAL_LEASTSQ_H_ + +#include "benchmark/benchmark_api.h" + +#include + +// This data structure will contain the result returned vy minimalLeastSq +// - coef : Estimated coeficient for the high-order term as interpolated from data. +// - rms : Normalized Root Mean Squared Error. +// - complexity : Scalability form (e.g. O_N, O_N_log_N). In case a scalability form has been provided to minimalLeastSq +// this will return the same value. In case BigO::O_Auto has been selected, this parameter will return the +// best fitting curve detected. + +struct LeastSq { + LeastSq() : + coef(0), + rms(0), + complexity(benchmark::O_None) {} + + double coef; + double rms; + benchmark::BigO complexity; +}; + +// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. +LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity = benchmark::O_Auto); + +#endif diff --git a/src/reporter.cc b/src/reporter.cc index fd97aba..dc7b76b 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" +#include "minimal_leastsq.h" #include #include diff --git a/test/complexity_test.cc b/test/complexity_test.cc index afa82ed..54a6cff 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -38,15 +38,6 @@ static void BM_Complexity_O_N(benchmark::State& state) { } BENCHMARK(BM_Complexity_O_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_N); BENCHMARK(BM_Complexity_O_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_Auto); - -static void BM_Complexity_O_M_plus_N(benchmark::State& state) { - std::string s1(state.range_x(), '-'); - std::string s2(state.range_x(), '-'); - while (state.KeepRunning()) - benchmark::DoNotOptimize(s1.compare(s2)); -} -BENCHMARK(BM_Complexity_O_M_plus_N) - ->RangeMultiplier(2)->Range(1<<10, 1<<18) -> Complexity(benchmark::O_M_plus_N); static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); -- cgit v1.2.3 From 2e5c397b4829503a5cb023ac67d2a1f13ebda3aa Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 21 May 2016 08:55:43 +0200 Subject: implemented complexity reporting --- include/benchmark/reporter.h | 11 +++++++-- src/console_reporter.cc | 19 ++++++++++++-- src/json_reporter.cc | 10 ++++++++ src/minimal_leastsq.cc | 6 ++--- src/minimal_leastsq.h | 2 +- src/reporter.cc | 59 ++++++++++++++++++++++++++++++++++---------- test/complexity_test.cc | 17 +++++++------ 7 files changed, 95 insertions(+), 29 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index b398800..24c2691 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -49,9 +49,11 @@ class BenchmarkReporter { bytes_per_second(0), items_per_second(0), max_heapbytes_used(0), - complexity(O_1), + complexity(O_None), arg1(0), - arg2(0) {} + arg2(0), + report_bigO(false), + report_rms(false) {} std::string benchmark_name; std::string report_label; // Empty if not set by benchmark. @@ -71,6 +73,10 @@ class BenchmarkReporter { BigO complexity; int arg1; int arg2; + + // Inform print function if the current run is a complexity report + bool report_bigO; + bool report_rms; }; // Called once for every suite of benchmarks run. @@ -102,6 +108,7 @@ protected: static void ComputeStats(const std::vector & reports, Run& mean, Run& stddev); static void ComputeBigO(const std::vector & reports, Run& bigO, Run& rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); + static std::string GetBigO(BigO complexity); }; // Simple reporter that outputs benchmark data to the console. This is the diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 0d8ab1d..b0b4130 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -88,7 +88,7 @@ void ConsoleReporter::ReportComplexity(const std::vector & complexity_repor Run bigO_data; Run rms_data; BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); - + // Output using PrintRun. PrintRunData(bigO_data); PrintRunData(rms_data); @@ -115,7 +115,22 @@ void ConsoleReporter::PrintRunData(const Run& result) { ColorPrintf(COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); - if (result.iterations == 0) { + if(result.report_bigO) { + std::string big_o = result.report_bigO ? GetBigO(result.complexity) : ""; + ColorPrintf(COLOR_YELLOW, "%10.4f %s %10.4f %s ", + result.real_accumulated_time * multiplier, + big_o.c_str(), + result.cpu_accumulated_time * multiplier, + big_o.c_str()); + } + else if(result.report_rms) { + ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", + result.real_accumulated_time * multiplier * 100, + "%", + result.cpu_accumulated_time * multiplier * 100, + "%"); + } + else if (result.iterations == 0) { ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, timeLabel, diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 07fc366..4874fe7 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -121,13 +121,23 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) return; } + std::string indent(4, ' '); + std::ostream& out = std::cout; + if (!first_report_) { + out << ",\n"; + } + Run bigO_data; Run rms_data; BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); // Output using PrintRun. + out << indent << "{\n"; PrintRunData(bigO_data); + out << indent << "},\n"; + out << indent << "{\n"; PrintRunData(rms_data); + out << indent << '}'; } void JSONReporter::Finalize() { diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index c4627d3..32d2745 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -41,7 +41,7 @@ double fittingCurve(double N, benchmark::BigO Complexity) { // - Complexity : Fitting curve. // For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -LeastSq leastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { +LeastSq leastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { assert(N.size() == Time.size() && N.size() >= 2); assert(Complexity != benchmark::O_None && Complexity != benchmark::O_Auto); @@ -79,7 +79,7 @@ LeastSq leastSq(const std::vector& N, const std::vector& Time, const b double mean = sigmaTime / N.size(); - result.rms = sqrt(rms) / mean; // Normalized RMS by the mean of the observed values + result.rms = sqrt(rms / N.size()) / mean; // Normalized RMS by the mean of the observed values return result; } @@ -90,7 +90,7 @@ LeastSq leastSq(const std::vector& N, const std::vector& Time, const b // - Complexity : If different than O_Auto, the fitting curve will stick to this one. If it is O_Auto, it will be calculated // the best fitting curve. -LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { +LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { assert(N.size() == Time.size() && N.size() >= 2); // Do not compute fitting curve is less than two benchmark runs are given assert(Complexity != benchmark::O_None); // Check that complexity is a valid parameter. diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index ae725d1..d0d5822 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -41,6 +41,6 @@ struct LeastSq { }; // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity = benchmark::O_Auto); +LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity = benchmark::O_Auto); #endif diff --git a/src/reporter.cc b/src/reporter.cc index dc7b76b..da40db6 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -17,6 +17,7 @@ #include #include +#include #include "check.h" #include "stat.h" @@ -83,35 +84,67 @@ void BenchmarkReporter::ComputeBigO( Run& bigO, Run& rms) { CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; // Accumulators. - Stat1_d real_accumulated_time_stat; - Stat1_d cpu_accumulated_time_stat; + std::vector N; + std::vector RealTime; + std::vector CpuTime; // Populate the accumulators. for (Run const& run : reports) { - real_accumulated_time_stat += - Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); - cpu_accumulated_time_stat += - Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); + N.push_back(run.arg1); + RealTime.push_back(run.real_accumulated_time/run.iterations); + CpuTime.push_back(run.cpu_accumulated_time/run.iterations); } + + LeastSq resultCpu = minimalLeastSq(N, CpuTime, reports[0].complexity); + + // resultCpu.complexity is passed as parameter to resultReal because in case + // reports[0].complexity is O_Auto, the noise on the measured data could make + // the best fit function of Cpu and Real differ. In order to solve this, we take + // the best fitting function for the Cpu, and apply it to Real data. + LeastSq resultReal = minimalLeastSq(N, RealTime, resultCpu.complexity); std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. bigO.benchmark_name = benchmark_name + "_BigO"; bigO.iterations = 0; - bigO.real_accumulated_time = real_accumulated_time_stat.Mean(); - bigO.cpu_accumulated_time = cpu_accumulated_time_stat.Mean(); + bigO.real_accumulated_time = resultReal.coef; + bigO.cpu_accumulated_time = resultCpu.coef; + bigO.report_bigO = true; + bigO.complexity = resultCpu.complexity; + + double multiplier; + const char* timeLabel; + std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(reports[0].time_unit); // Only add label to mean/stddev if it is same for all runs bigO.report_label = reports[0].report_label; - rms.benchmark_name = benchmark_name + "_RMS"; rms.report_label = bigO.report_label; rms.iterations = 0; - rms.real_accumulated_time = - real_accumulated_time_stat.StdDev(); - rms.cpu_accumulated_time = - cpu_accumulated_time_stat.StdDev(); + rms.real_accumulated_time = resultReal.rms / multiplier; + rms.cpu_accumulated_time = resultCpu.rms / multiplier; + rms.report_rms = true; + rms.complexity = resultCpu.complexity; +} + +std::string BenchmarkReporter::GetBigO(BigO complexity) { + switch (complexity) { + case O_N: + return "* N"; + case O_N_Squared: + return "* N**2"; + case O_N_Cubed: + return "* N**3"; + case O_log_N: + return "* lgN"; + case O_N_log_N: + return "* NlgN"; + case O_1: + return "* 1"; + default: + return ""; + } } TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) { diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 54a6cff..321fdad 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -27,7 +27,7 @@ void BM_Complexity_O1(benchmark::State& state) { while (state.KeepRunning()) { } } -BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<17) -> Complexity(benchmark::O_1); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::O_1); static void BM_Complexity_O_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); @@ -36,9 +36,9 @@ static void BM_Complexity_O_N(benchmark::State& state) { benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector)); } } -BENCHMARK(BM_Complexity_O_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_N); -BENCHMARK(BM_Complexity_O_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_Auto); - +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_N); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); + static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); std::string s2(state.range_x(), '-'); @@ -76,7 +76,8 @@ static void BM_Complexity_O_log_N(benchmark::State& state) { benchmark::DoNotOptimize(m.find(itemNotInVector)); } } -BENCHMARK(BM_Complexity_O_log_N) -> Range(1, 1<<10) -> Complexity(benchmark::O_log_N); +BENCHMARK(BM_Complexity_O_log_N) + ->RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_log_N); static void BM_Complexity_O_N_log_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); @@ -84,10 +85,10 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { std::sort(v.begin(), v.end()); } } -BENCHMARK(BM_Complexity_O_N_log_N) -> Range(1, 1<<16) -> Complexity(benchmark::O_N_log_N); -BENCHMARK(BM_Complexity_O_N_log_N) -> Range(1, 1<<16) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_N_log_N); +BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); -// Test benchmark with no range. Complexity is always calculated as O(1). +// Test benchmark with no range and check no complexity is calculated. void BM_Extreme_Cases(benchmark::State& state) { while (state.KeepRunning()) { } -- cgit v1.2.3 From 290bd60289ef571875415cf82be805f9a446c6a9 Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 21 May 2016 11:51:42 +0200 Subject: Refactor for pull request --- AUTHORS | 1 + CONTRIBUTORS | 1 + include/benchmark/benchmark_api.h | 7 ++++--- include/benchmark/reporter.h | 2 +- src/minimal_leastsq.cc | 2 +- src/minimal_leastsq.h | 4 ++-- src/reporter.cc | 4 ++-- test/complexity_test.cc | 12 ++++++------ 8 files changed, 18 insertions(+), 15 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9da43c7..7ddffd8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -16,6 +16,7 @@ Eugene Zhuk Evgeny Safronov Felix Homann Google Inc. +Ismael Jimenez Martinez JianXiong Zhou Jussi Knuuttila Kaito Udagawa diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 67ecb28..a575ef1 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -31,6 +31,7 @@ Dominic Hamon Eugene Zhuk Evgeny Safronov Felix Homann +Ismael Jimenez Martinez JianXiong Zhou Jussi Knuuttila Kaito Udagawa diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 146a8cc..d7cf83f 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -232,7 +232,8 @@ enum TimeUnit { }; // BigO is passed to a benchmark in order to specify the asymptotic computational -// complexity for the benchmark. +// complexity for the benchmark. In case O_Auto is selected, complexity will be +// calculated automatically to the best fit. enum BigO { O_None, O_1, @@ -479,8 +480,8 @@ public: // or MB/second values. Benchmark* UseManualTime(); - // Set the asymptotic computational complexity for the benchmark. This option - // called the asymptotic computational complexity will be shown on the output. + // Set the asymptotic computational complexity for the benchmark. If called + // the asymptotic computational complexity will be shown on the output. Benchmark* Complexity(BigO complexity); // Support for running multiple copies of the same benchmark concurrently diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 24c2691..d6b713a 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -74,7 +74,7 @@ class BenchmarkReporter { int arg1; int arg2; - // Inform print function if the current run is a complexity report + // Inform print function whether the current run is a complexity report bool report_bigO; bool report_rms; }; diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index 32d2745..07a47b9 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -13,7 +13,7 @@ // limitations under the License. // Source project : https://github.com/ismaelJimenez/cpp.leastsq -// Addapted to be used with google benchmark +// Adapted to be used with google benchmark #include "minimal_leastsq.h" diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index d0d5822..0b137fb 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -13,7 +13,7 @@ // limitations under the License. // Source project : https://github.com/ismaelJimenez/cpp.leastsq -// Addapted to be used with google benchmark +// Adapted to be used with google benchmark #if !defined(MINIMAL_LEASTSQ_H_) #define MINIMAL_LEASTSQ_H_ @@ -22,7 +22,7 @@ #include -// This data structure will contain the result returned vy minimalLeastSq +// This data structure will contain the result returned by minimalLeastSq // - coef : Estimated coeficient for the high-order term as interpolated from data. // - rms : Normalized Root Mean Squared Error. // - complexity : Scalability form (e.g. O_N, O_N_log_N). In case a scalability form has been provided to minimalLeastSq diff --git a/src/reporter.cc b/src/reporter.cc index da40db6..61a6d5c 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -85,11 +85,11 @@ void BenchmarkReporter::ComputeBigO( CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; // Accumulators. std::vector N; - std::vector RealTime; + std::vector RealTime; std::vector CpuTime; // Populate the accumulators. - for (Run const& run : reports) { + for (const Run& run : reports) { N.push_back(run.arg1); RealTime.push_back(run.real_accumulated_time/run.iterations); CpuTime.push_back(run.cpu_accumulated_time/run.iterations); diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 321fdad..e7e16d3 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -36,8 +36,8 @@ static void BM_Complexity_O_N(benchmark::State& state) { benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector)); } } -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_N); -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_N); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); @@ -77,7 +77,7 @@ static void BM_Complexity_O_log_N(benchmark::State& state) { } } BENCHMARK(BM_Complexity_O_log_N) - ->RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_log_N); + -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_log_N); static void BM_Complexity_O_N_log_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); @@ -85,8 +85,8 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { std::sort(v.begin(), v.end()); } } -BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_N_log_N); -BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2)->Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_N_log_N); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); // Test benchmark with no range and check no complexity is calculated. void BM_Extreme_Cases(benchmark::State& state) { @@ -94,6 +94,6 @@ void BM_Extreme_Cases(benchmark::State& state) { } } BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N); -BENCHMARK(BM_Extreme_Cases)->Arg(42) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::O_Auto); BENCHMARK_MAIN() \ No newline at end of file -- cgit v1.2.3 From 5812d545efcd3bcbfd4e2e8d203bd42b8e5a7148 Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 21 May 2016 12:16:40 +0200 Subject: Added range multiplier to Readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 051b301..325675a 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,13 @@ the specified range and will generate a benchmark for each such argument. BENCHMARK(BM_memcpy)->Range(8, 8<<10); ``` +By default the arguments in a range are generated in multiples of eight and the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the range multiplier is changed to multiples of two. + +```c++ +BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); +``` +Now the arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. + You might have a benchmark that depends on two inputs. For example, the following code defines a family of benchmarks for measuring the speed of set insertion. -- cgit v1.2.3 From dc667d048678f3cb6b4355d4da8b5d121db8bbf2 Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 21 May 2016 12:40:27 +0200 Subject: Added asymptotic complexity to Readme --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 325675a..f052cb8 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,27 @@ static void CustomArguments(benchmark::internal::Benchmark* b) { BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` +### Calculate asymptotic complexity (Big O) +Asymptotic complexity might be calculated for a family of benchmarks. The following code will calculate the coefficient for the high-order term in the running time and the normalized root-mean square error of string comparison. + +```c++ +static void BM_StringCompare(benchmark::State& state) { + std::string s1(state.range_x(), '-'); + std::string s2(state.range_x(), '-'); + while (state.KeepRunning()) + benchmark::DoNotOptimize(s1.compare(s2)); +} +BENCHMARK(BM_StringCompare) + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_N); +``` + +As shown on the following invocation, asymptotic complexity might also be calculated automatically. + +```c++ +BENCHMARK(BM_StringCompare) + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_Auto); +``` + ### Templated benchmarks Templated benchmarks work the same way: This example produces and consumes messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the -- cgit v1.2.3 From 07efafbf5c0e95fabbae284eedcc1dfe3d57a396 Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 21 May 2016 16:34:12 +0200 Subject: Update Readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f052cb8..cd78c96 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,12 @@ the specified range and will generate a benchmark for each such argument. BENCHMARK(BM_memcpy)->Range(8, 8<<10); ``` -By default the arguments in a range are generated in multiples of eight and the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the range multiplier is changed to multiples of two. +By default the arguments in the range are generated in multiples of eight and the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the range multiplier is changed to multiples of two. ```c++ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); ``` -Now the arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. +Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. You might have a benchmark that depends on two inputs. For example, the following code defines a family of benchmarks for measuring the speed of set @@ -130,7 +130,7 @@ BENCHMARK(BM_StringCompare) ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_N); ``` -As shown on the following invocation, asymptotic complexity might also be calculated automatically. +As shown in the following invocation, asymptotic complexity might also be calculated automatically. ```c++ BENCHMARK(BM_StringCompare) -- cgit v1.2.3 From 8afbf0ed3801ad12c4066d10e9d25764181321f4 Mon Sep 17 00:00:00 2001 From: Ismael Date: Sat, 21 May 2016 16:45:45 +0200 Subject: reworked comment for complexity report --- include/benchmark/reporter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index d6b713a..3bf45d8 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -93,8 +93,8 @@ class BenchmarkReporter { // benchmark, thus have the same name. virtual void ReportRuns(const std::vector& report) = 0; - // Called once at the last instance of a benchmark range, gives information about - // asymptotic complexity and RMS. + // Called once at the last benchmark in a family of benchmarks, gives information + // about asymptotic complexity and RMS. // Note that all the benchmark runs in a range should refer to the same benchmark, // thus have the same name. virtual void ReportComplexity(const std::vector& complexity_reports) = 0; -- cgit v1.2.3 From 5f9823bd92b2a24da06fac7b43f6658ec20cc901 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 18:51:29 +0200 Subject: fixed non-const reference arguments --- include/benchmark/reporter.h | 8 +++--- src/console_reporter.cc | 12 ++++----- src/csv_reporter.cc | 4 +-- src/json_reporter.cc | 4 +-- src/reporter.cc | 62 ++++++++++++++++++++++---------------------- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 3bf45d8..564219a 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -52,7 +52,7 @@ class BenchmarkReporter { complexity(O_None), arg1(0), arg2(0), - report_bigO(false), + report_big_o(false), report_rms(false) {} std::string benchmark_name; @@ -75,7 +75,7 @@ class BenchmarkReporter { int arg2; // Inform print function whether the current run is a complexity report - bool report_bigO; + bool report_big_o; bool report_rms; }; @@ -105,8 +105,8 @@ class BenchmarkReporter { virtual ~BenchmarkReporter(); protected: - static void ComputeStats(const std::vector & reports, Run& mean, Run& stddev); - static void ComputeBigO(const std::vector & reports, Run& bigO, Run& rms); + static void ComputeStats(const std::vector & reports, Run* mean, Run* stddev); + static void ComputeBigO(const std::vector & reports, Run* bigO, Run* rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); static std::string GetBigO(BigO complexity); }; diff --git a/src/console_reporter.cc b/src/console_reporter.cc index b0b4130..09b91c2 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -72,7 +72,7 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); + BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); // Output using PrintRun. PrintRunData(mean_data); @@ -85,12 +85,12 @@ void ConsoleReporter::ReportComplexity(const std::vector & complexity_repor return; } - Run bigO_data; + Run big_o_data; Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); // Output using PrintRun. - PrintRunData(bigO_data); + PrintRunData(big_o_data); PrintRunData(rms_data); } @@ -115,8 +115,8 @@ void ConsoleReporter::PrintRunData(const Run& result) { ColorPrintf(COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); - if(result.report_bigO) { - std::string big_o = result.report_bigO ? GetBigO(result.complexity) : ""; + if(result.report_big_o) { + std::string big_o = result.report_big_o ? GetBigO(result.complexity) : ""; ColorPrintf(COLOR_YELLOW, "%10.4f %s %10.4f %s ", result.real_accumulated_time * multiplier, big_o.c_str(), diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index f13a5f8..031736e 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -57,7 +57,7 @@ void CSVReporter::ReportRuns(const std::vector & reports) { if (reports.size() >= 2) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); + BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); reports_cp.push_back(mean_data); reports_cp.push_back(stddev_data); } @@ -74,7 +74,7 @@ void CSVReporter::ReportComplexity(const std::vector & complexity_reports) Run bigO_data; Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + BenchmarkReporter::ComputeBigO(complexity_reports, &bigO_data, &rms_data); // Output using PrintRun. PrintRunData(bigO_data); diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 4874fe7..c15fb10 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -100,7 +100,7 @@ void JSONReporter::ReportRuns(std::vector const& reports) { if (reports.size() >= 2) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); + BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); reports_cp.push_back(mean_data); reports_cp.push_back(stddev_data); } @@ -129,7 +129,7 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) Run bigO_data; Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + BenchmarkReporter::ComputeBigO(complexity_reports, &bigO_data, &rms_data); // Output using PrintRun. out << indent << "{\n"; diff --git a/src/reporter.cc b/src/reporter.cc index 61a6d5c..27dca85 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -26,7 +26,7 @@ namespace benchmark { void BenchmarkReporter::ComputeStats( const std::vector& reports, - Run& mean_data, Run& stddev_data) { + Run* mean_data, Run* stddev_data) { CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports"; // Accumulators. Stat1_d real_accumulated_time_stat; @@ -50,38 +50,38 @@ void BenchmarkReporter::ComputeStats( } // Get the data from the accumulator to BenchmarkReporter::Run's. - mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; - mean_data.iterations = run_iterations; - mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * + mean_data->benchmark_name = reports[0].benchmark_name + "_mean"; + mean_data->iterations = run_iterations; + mean_data->real_accumulated_time = real_accumulated_time_stat.Mean() * run_iterations; - mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * + mean_data->cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * run_iterations; - mean_data.bytes_per_second = bytes_per_second_stat.Mean(); - mean_data.items_per_second = items_per_second_stat.Mean(); + mean_data->bytes_per_second = bytes_per_second_stat.Mean(); + mean_data->items_per_second = items_per_second_stat.Mean(); // Only add label to mean/stddev if it is same for all runs - mean_data.report_label = reports[0].report_label; + mean_data->report_label = reports[0].report_label; for (std::size_t i = 1; i < reports.size(); i++) { if (reports[i].report_label != reports[0].report_label) { - mean_data.report_label = ""; + mean_data->report_label = ""; break; } } - stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; - stddev_data.report_label = mean_data.report_label; - stddev_data.iterations = 0; - stddev_data.real_accumulated_time = + stddev_data->benchmark_name = reports[0].benchmark_name + "_stddev"; + stddev_data->report_label = mean_data->report_label; + stddev_data->iterations = 0; + stddev_data->real_accumulated_time = real_accumulated_time_stat.StdDev(); - stddev_data.cpu_accumulated_time = + stddev_data->cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); - stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); - stddev_data.items_per_second = items_per_second_stat.StdDev(); + stddev_data->bytes_per_second = bytes_per_second_stat.StdDev(); + stddev_data->items_per_second = items_per_second_stat.StdDev(); } void BenchmarkReporter::ComputeBigO( const std::vector& reports, - Run& bigO, Run& rms) { + Run* big_o, Run* rms) { CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; // Accumulators. std::vector N; @@ -106,26 +106,26 @@ void BenchmarkReporter::ComputeBigO( std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. - bigO.benchmark_name = benchmark_name + "_BigO"; - bigO.iterations = 0; - bigO.real_accumulated_time = resultReal.coef; - bigO.cpu_accumulated_time = resultCpu.coef; - bigO.report_bigO = true; - bigO.complexity = resultCpu.complexity; + big_o->benchmark_name = benchmark_name + "_BigO"; + big_o->iterations = 0; + big_o->real_accumulated_time = resultReal.coef; + big_o->cpu_accumulated_time = resultCpu.coef; + big_o->report_big_o = true; + big_o->complexity = resultCpu.complexity; double multiplier; const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(reports[0].time_unit); // Only add label to mean/stddev if it is same for all runs - bigO.report_label = reports[0].report_label; - rms.benchmark_name = benchmark_name + "_RMS"; - rms.report_label = bigO.report_label; - rms.iterations = 0; - rms.real_accumulated_time = resultReal.rms / multiplier; - rms.cpu_accumulated_time = resultCpu.rms / multiplier; - rms.report_rms = true; - rms.complexity = resultCpu.complexity; + big_o->report_label = reports[0].report_label; + rms->benchmark_name = benchmark_name + "_RMS"; + rms->report_label = big_o->report_label; + rms->iterations = 0; + rms->real_accumulated_time = resultReal.rms / multiplier; + rms->cpu_accumulated_time = resultCpu.rms / multiplier; + rms->report_rms = true; + rms->complexity = resultCpu.complexity; } std::string BenchmarkReporter::GetBigO(BigO complexity) { -- cgit v1.2.3 From 5e52d2d6c048205626c4103d397bedf0527f67d8 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 19:19:29 +0200 Subject: refactor fitting curve --- src/minimal_leastsq.cc | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index 07a47b9..f3c841d 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -20,19 +20,22 @@ #include // Internal function to calculate the different scalability forms -double fittingCurve(double N, benchmark::BigO Complexity) { - if (Complexity == benchmark::O_N) - return N; - else if (Complexity == benchmark::O_N_Squared) - return pow(N, 2); - else if (Complexity == benchmark::O_N_Cubed) - return pow(N, 3); - else if (Complexity == benchmark::O_log_N) - return log2(N); - else if (Complexity == benchmark::O_N_log_N) - return N * log2(N); - - return 1; // Default value for O_1 +double fittingCurve(double n, benchmark::BigO complexity) { + switch (complexity) { + case benchmark::O_N: + return n; + case benchmark::O_N_Squared: + return pow(n, 2); + case benchmark::O_N_Cubed: + return pow(n, 3); + case benchmark::O_log_N: + return log2(n); + case benchmark::O_N_log_N: + return n * log2(n); + case benchmark::O_1: + default: + return 1; + } } // Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -- cgit v1.2.3 From ac05c045335d3e32ec75e3aae930ecc1c6533212 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 20:12:54 +0200 Subject: refactor MinimalLEastSq --- README.md | 4 +- include/benchmark/benchmark_api.h | 18 ++++---- include/benchmark/reporter.h | 2 +- src/benchmark.cc | 6 +-- src/csv_reporter.cc | 6 +-- src/json_reporter.cc | 6 +-- src/minimal_leastsq.cc | 93 +++++++++++++++++++-------------------- src/minimal_leastsq.h | 10 ++--- src/reporter.cc | 18 ++++---- test/complexity_test.cc | 20 ++++----- 10 files changed, 91 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index cd78c96..c989e57 100644 --- a/README.md +++ b/README.md @@ -127,14 +127,14 @@ static void BM_StringCompare(benchmark::State& state) { benchmark::DoNotOptimize(s1.compare(s2)); } BENCHMARK(BM_StringCompare) - ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_N); + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN); ``` As shown in the following invocation, asymptotic complexity might also be calculated automatically. ```c++ BENCHMARK(BM_StringCompare) - ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_Auto); + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oAuto); ``` ### Templated benchmarks diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index d7cf83f..385cbbc 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -232,17 +232,17 @@ enum TimeUnit { }; // BigO is passed to a benchmark in order to specify the asymptotic computational -// complexity for the benchmark. In case O_Auto is selected, complexity will be +// complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. enum BigO { - O_None, - O_1, - O_N, - O_N_Squared, - O_N_Cubed, - O_log_N, - O_N_log_N, - O_Auto + oNone, + o1, + oN, + oNSquared, + oNCubed, + oLogN, + oNLogN, + oAuto }; // State is passed to a running Benchmark and contains state for the diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 564219a..5e0a552 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -49,7 +49,7 @@ class BenchmarkReporter { bytes_per_second(0), items_per_second(0), max_heapbytes_used(0), - complexity(O_None), + complexity(oNone), arg1(0), arg2(0), report_big_o(false), diff --git a/src/benchmark.cc b/src/benchmark.cc index 874dc0c..dbab503 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -454,7 +454,7 @@ BenchmarkImp::BenchmarkImp(const char* name) : name_(name), arg_count_(-1), time_unit_(kNanosecond), range_multiplier_(kRangeMultiplier), min_time_(0.0), use_real_time_(false), use_manual_time_(false), - complexity_(O_None) { + complexity_(oNone) { } BenchmarkImp::~BenchmarkImp() { @@ -803,7 +803,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.complexity = b.complexity; reports.push_back(report); - if(report.complexity != O_None) + if(report.complexity != oNone) complexity_reports.push_back(report); break; @@ -830,7 +830,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, } br->ReportRuns(reports); - if((b.complexity != O_None) && b.last_benchmark_instance) { + if((b.complexity != oNone) && b.last_benchmark_instance) { br->ReportComplexity(complexity_reports); complexity_reports.clear(); } diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 031736e..df662e4 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -72,12 +72,12 @@ void CSVReporter::ReportComplexity(const std::vector & complexity_reports) return; } - Run bigO_data; + Run big_o_data; Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, &bigO_data, &rms_data); + BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); // Output using PrintRun. - PrintRunData(bigO_data); + PrintRunData(big_o_data); PrintRunData(rms_data); } diff --git a/src/json_reporter.cc b/src/json_reporter.cc index c15fb10..bfa85e4 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -127,13 +127,13 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) out << ",\n"; } - Run bigO_data; + Run big_o_data; Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, &bigO_data, &rms_data); + BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); // Output using PrintRun. out << indent << "{\n"; - PrintRunData(bigO_data); + PrintRunData(big_o_data); out << indent << "},\n"; out << indent << "{\n"; PrintRunData(rms_data); diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index f3c841d..0b6e665 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -16,95 +16,94 @@ // Adapted to be used with google benchmark #include "minimal_leastsq.h" - +#include "check.h" #include // Internal function to calculate the different scalability forms -double fittingCurve(double n, benchmark::BigO complexity) { +double FittingCurve(double n, benchmark::BigO complexity) { switch (complexity) { - case benchmark::O_N: + case benchmark::oN: return n; - case benchmark::O_N_Squared: + case benchmark::oNSquared: return pow(n, 2); - case benchmark::O_N_Cubed: + case benchmark::oNCubed: return pow(n, 3); - case benchmark::O_log_N: + case benchmark::oLogN: return log2(n); - case benchmark::O_N_log_N: + case benchmark::oNLogN: return n * log2(n); - case benchmark::O_1: + case benchmark::o1: default: return 1; } } // Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -// - N : Vector containing the size of the benchmark tests. -// - Time : Vector containing the times for the benchmark tests. -// - Complexity : Fitting curve. +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - complexity : Fitting curve. // For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -LeastSq leastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { - assert(N.size() == Time.size() && N.size() >= 2); - assert(Complexity != benchmark::O_None && - Complexity != benchmark::O_Auto); +LeastSq CalculateLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity) { + CHECK_NE(complexity, benchmark::oAuto); - double sigmaGN = 0; - double sigmaGNSquared = 0; - double sigmaTime = 0; - double sigmaTimeGN = 0; + double sigma_gn = 0; + double sigma_gn_squared = 0; + double sigma_time = 0; + double sigma_time_gn = 0; // Calculate least square fitting parameter - for (size_t i = 0; i < N.size(); ++i) { - double GNi = fittingCurve(N[i], Complexity); - sigmaGN += GNi; - sigmaGNSquared += GNi * GNi; - sigmaTime += Time[i]; - sigmaTimeGN += Time[i] * GNi; + for (size_t i = 0; i < n.size(); ++i) { + double gn_i = FittingCurve(n[i], complexity); + sigma_gn += gn_i; + sigma_gn_squared += gn_i * gn_i; + sigma_time += time[i]; + sigma_time_gn += time[i] * gn_i; } LeastSq result; - result.complexity = Complexity; + result.complexity = complexity; // Calculate complexity. - // O_1 is treated as an special case - if (Complexity != benchmark::O_1) - result.coef = sigmaTimeGN / sigmaGNSquared; + // o1 is treated as an special case + if (complexity != benchmark::o1) + result.coef = sigma_time_gn / sigma_gn_squared; else - result.coef = sigmaTime / N.size(); + result.coef = sigma_time / n.size(); // Calculate RMS double rms = 0; - for (size_t i = 0; i < N.size(); ++i) { - double fit = result.coef * fittingCurve(N[i], Complexity); - rms += pow((Time[i] - fit), 2); + for (size_t i = 0; i < n.size(); ++i) { + double fit = result.coef * FittingCurve(n[i], complexity); + rms += pow((time[i] - fit), 2); } - double mean = sigmaTime / N.size(); + double mean = sigma_time / n.size(); - result.rms = sqrt(rms / N.size()) / mean; // Normalized RMS by the mean of the observed values + result.rms = sqrt(rms / n.size()) / mean; // Normalized RMS by the mean of the observed values return result; } // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -// - N : Vector containing the size of the benchmark tests. -// - Time : Vector containing the times for the benchmark tests. -// - Complexity : If different than O_Auto, the fitting curve will stick to this one. If it is O_Auto, it will be calculated +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - complexity : If different than oAuto, the fitting curve will stick to this one. If it is oAuto, it will be calculated // the best fitting curve. -LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity) { - assert(N.size() == Time.size() && N.size() >= 2); // Do not compute fitting curve is less than two benchmark runs are given - assert(Complexity != benchmark::O_None); // Check that complexity is a valid parameter. +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity) { + CHECK_EQ(n.size(), time.size()); + CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given + CHECK_NE(complexity, benchmark::oNone); - if(Complexity == benchmark::O_Auto) { - std::vector fitCurves = { benchmark::O_log_N, benchmark::O_N, benchmark::O_N_log_N, benchmark::O_N_Squared, benchmark::O_N_Cubed }; + if(complexity == benchmark::oAuto) { + std::vector fit_curves = { benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, benchmark::oNCubed }; - LeastSq best_fit = leastSq(N, Time, benchmark::O_1); // Take O_1 as default best fitting curve + LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); // Take o1 as default best fitting curve // Compute all possible fitting curves and stick to the best one - for (const auto& fit : fitCurves) { - LeastSq current_fit = leastSq(N, Time, fit); + for (const auto& fit : fit_curves) { + LeastSq current_fit = CalculateLeastSq(n, time, fit); if (current_fit.rms < best_fit.rms) best_fit = current_fit; } @@ -112,5 +111,5 @@ LeastSq minimalLeastSq(const std::vector& N, const std::vector& Tim return best_fit; } else - return leastSq(N, Time, Complexity); + return CalculateLeastSq(n, time, complexity); } \ No newline at end of file diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index 0b137fb..3f7be34 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -22,18 +22,18 @@ #include -// This data structure will contain the result returned by minimalLeastSq +// This data structure will contain the result returned by MinimalLeastSq // - coef : Estimated coeficient for the high-order term as interpolated from data. // - rms : Normalized Root Mean Squared Error. -// - complexity : Scalability form (e.g. O_N, O_N_log_N). In case a scalability form has been provided to minimalLeastSq -// this will return the same value. In case BigO::O_Auto has been selected, this parameter will return the +// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability form has been provided to MinimalLeastSq +// this will return the same value. In case BigO::oAuto has been selected, this parameter will return the // best fitting curve detected. struct LeastSq { LeastSq() : coef(0), rms(0), - complexity(benchmark::O_None) {} + complexity(benchmark::oNone) {} double coef; double rms; @@ -41,6 +41,6 @@ struct LeastSq { }; // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -LeastSq minimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity = benchmark::O_Auto); +LeastSq MinimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity = benchmark::oAuto); #endif diff --git a/src/reporter.cc b/src/reporter.cc index 27dca85..0e1c581 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -95,13 +95,13 @@ void BenchmarkReporter::ComputeBigO( CpuTime.push_back(run.cpu_accumulated_time/run.iterations); } - LeastSq resultCpu = minimalLeastSq(N, CpuTime, reports[0].complexity); + LeastSq resultCpu = MinimalLeastSq(N, CpuTime, reports[0].complexity); // resultCpu.complexity is passed as parameter to resultReal because in case - // reports[0].complexity is O_Auto, the noise on the measured data could make + // reports[0].complexity is oAuto, the noise on the measured data could make // the best fit function of Cpu and Real differ. In order to solve this, we take // the best fitting function for the Cpu, and apply it to Real data. - LeastSq resultReal = minimalLeastSq(N, RealTime, resultCpu.complexity); + LeastSq resultReal = MinimalLeastSq(N, RealTime, resultCpu.complexity); std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); @@ -130,17 +130,17 @@ void BenchmarkReporter::ComputeBigO( std::string BenchmarkReporter::GetBigO(BigO complexity) { switch (complexity) { - case O_N: + case oN: return "* N"; - case O_N_Squared: + case oNSquared: return "* N**2"; - case O_N_Cubed: + case oNCubed: return "* N**3"; - case O_log_N: + case oLogN: return "* lgN"; - case O_N_log_N: + case oNLogN: return "* NlgN"; - case O_1: + case o1: return "* 1"; default: return ""; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index e7e16d3..e81742e 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -27,7 +27,7 @@ void BM_Complexity_O1(benchmark::State& state) { while (state.KeepRunning()) { } } -BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::O_1); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); static void BM_Complexity_O_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); @@ -36,8 +36,8 @@ static void BM_Complexity_O_N(benchmark::State& state) { benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector)); } } -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_N); -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); @@ -50,7 +50,7 @@ static void BM_Complexity_O_N_Squared(benchmark::State& state) { } } } -BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::O_N_Squared); +BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::oNSquared); static void BM_Complexity_O_N_Cubed(benchmark::State& state) { std::string s1(state.range_x(), '-'); @@ -67,7 +67,7 @@ static void BM_Complexity_O_N_Cubed(benchmark::State& state) { } } } -BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::O_N_Cubed); +BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::oNCubed); static void BM_Complexity_O_log_N(benchmark::State& state) { auto m = ConstructRandomMap(state.range_x()); @@ -77,7 +77,7 @@ static void BM_Complexity_O_log_N(benchmark::State& state) { } } BENCHMARK(BM_Complexity_O_log_N) - -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_log_N); + -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oLogN); static void BM_Complexity_O_N_log_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); @@ -85,15 +85,15 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { std::sort(v.begin(), v.end()); } } -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_N_log_N); -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); // Test benchmark with no range and check no complexity is calculated. void BM_Extreme_Cases(benchmark::State& state) { while (state.KeepRunning()) { } } -BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N); -BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::O_Auto); +BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::oNLogN); +BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::oAuto); BENCHMARK_MAIN() \ No newline at end of file -- cgit v1.2.3 From 266addc3f51f07ec182ed34af06e6d75b0f6d09f Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 20:21:34 +0200 Subject: fixed last_benchmark_instance --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index dbab503..2245389 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -415,7 +415,6 @@ bool BenchmarkFamilies::FindBenchmarks( instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; - instance.last_benchmark_instance = (args == family->args_.back()); instance.complexity = family->complexity_; instance.threads = num_threads; instance.multithreaded = !(family->thread_counts_.empty()); @@ -442,6 +441,7 @@ bool BenchmarkFamilies::FindBenchmarks( } if (re.Match(instance.name)) { + instance.last_benchmark_instance = (args == family->args_.back()); benchmarks->push_back(instance); } } -- cgit v1.2.3 From fed9b6f211a9ca67860bcaf183cb450e3695bb07 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 20:34:01 +0200 Subject: refactor least square .h --- src/minimal_leastsq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index 3f7be34..8feb43d 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -41,6 +41,6 @@ struct LeastSq { }; // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -LeastSq MinimalLeastSq(const std::vector& N, const std::vector& Time, const benchmark::BigO Complexity = benchmark::oAuto); +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO vomplexity = benchmark::oAuto); #endif -- cgit v1.2.3 From ea69a8479046413d96b0eb826f1d982985281a67 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 20:34:54 +0200 Subject: fix --- src/minimal_leastsq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index 8feb43d..80d2633 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -41,6 +41,6 @@ struct LeastSq { }; // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO vomplexity = benchmark::oAuto); +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity = benchmark::oAuto); #endif -- cgit v1.2.3 From d577987fd76595cb52602bd75b2866886e95b0f2 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 20:40:41 +0200 Subject: changed indentation --- src/minimal_leastsq.cc | 146 ++++++++++++++++++++++++------------------------- src/minimal_leastsq.h | 16 +++--- 2 files changed, 81 insertions(+), 81 deletions(-) diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index 0b6e665..ea6bd46 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -21,21 +21,21 @@ // Internal function to calculate the different scalability forms double FittingCurve(double n, benchmark::BigO complexity) { - switch (complexity) { - case benchmark::oN: - return n; - case benchmark::oNSquared: - return pow(n, 2); - case benchmark::oNCubed: - return pow(n, 3); - case benchmark::oLogN: - return log2(n); - case benchmark::oNLogN: - return n * log2(n); - case benchmark::o1: - default: - return 1; - } + switch (complexity) { + case benchmark::oN: + return n; + case benchmark::oNSquared: + return pow(n, 2); + case benchmark::oNCubed: + return pow(n, 3); + case benchmark::oLogN: + return log2(n); + case benchmark::oNLogN: + return n * log2(n); + case benchmark::o1: + default: + return 1; + } } // Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. @@ -45,44 +45,44 @@ double FittingCurve(double n, benchmark::BigO complexity) { // For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit LeastSq CalculateLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity) { - CHECK_NE(complexity, benchmark::oAuto); - - double sigma_gn = 0; - double sigma_gn_squared = 0; - double sigma_time = 0; - double sigma_time_gn = 0; - - // Calculate least square fitting parameter - for (size_t i = 0; i < n.size(); ++i) { - double gn_i = FittingCurve(n[i], complexity); - sigma_gn += gn_i; - sigma_gn_squared += gn_i * gn_i; - sigma_time += time[i]; - sigma_time_gn += time[i] * gn_i; - } - - LeastSq result; - result.complexity = complexity; - - // Calculate complexity. - // o1 is treated as an special case - if (complexity != benchmark::o1) - result.coef = sigma_time_gn / sigma_gn_squared; - else - result.coef = sigma_time / n.size(); - - // Calculate RMS - double rms = 0; - for (size_t i = 0; i < n.size(); ++i) { - double fit = result.coef * FittingCurve(n[i], complexity); - rms += pow((time[i] - fit), 2); - } - - double mean = sigma_time / n.size(); - - result.rms = sqrt(rms / n.size()) / mean; // Normalized RMS by the mean of the observed values - - return result; + CHECK_NE(complexity, benchmark::oAuto); + + double sigma_gn = 0; + double sigma_gn_squared = 0; + double sigma_time = 0; + double sigma_time_gn = 0; + + // Calculate least square fitting parameter + for (size_t i = 0; i < n.size(); ++i) { + double gn_i = FittingCurve(n[i], complexity); + sigma_gn += gn_i; + sigma_gn_squared += gn_i * gn_i; + sigma_time += time[i]; + sigma_time_gn += time[i] * gn_i; + } + + LeastSq result; + result.complexity = complexity; + + // Calculate complexity. + // o1 is treated as an special case + if (complexity != benchmark::o1) + result.coef = sigma_time_gn / sigma_gn_squared; + else + result.coef = sigma_time / n.size(); + + // Calculate RMS + double rms = 0; + for (size_t i = 0; i < n.size(); ++i) { + double fit = result.coef * FittingCurve(n[i], complexity); + rms += pow((time[i] - fit), 2); + } + + double mean = sigma_time / n.size(); + + result.rms = sqrt(rms / n.size()) / mean; // Normalized RMS by the mean of the observed values + + return result; } // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. @@ -92,24 +92,24 @@ LeastSq CalculateLeastSq(const std::vector& n, const std::vector& t // the best fitting curve. LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity) { - CHECK_EQ(n.size(), time.size()); - CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given - CHECK_NE(complexity, benchmark::oNone); - - if(complexity == benchmark::oAuto) { - std::vector fit_curves = { benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, benchmark::oNCubed }; - - LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); // Take o1 as default best fitting curve - - // Compute all possible fitting curves and stick to the best one - for (const auto& fit : fit_curves) { - LeastSq current_fit = CalculateLeastSq(n, time, fit); - if (current_fit.rms < best_fit.rms) - best_fit = current_fit; - } - - return best_fit; - } - else - return CalculateLeastSq(n, time, complexity); + CHECK_EQ(n.size(), time.size()); + CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given + CHECK_NE(complexity, benchmark::oNone); + + if(complexity == benchmark::oAuto) { + std::vector fit_curves = { benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, benchmark::oNCubed }; + + LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); // Take o1 as default best fitting curve + + // Compute all possible fitting curves and stick to the best one + for (const auto& fit : fit_curves) { + LeastSq current_fit = CalculateLeastSq(n, time, fit); + if (current_fit.rms < best_fit.rms) + best_fit = current_fit; + } + + return best_fit; + } + else + return CalculateLeastSq(n, time, complexity); } \ No newline at end of file diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index 80d2633..6dcb894 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -30,14 +30,14 @@ // best fitting curve detected. struct LeastSq { - LeastSq() : - coef(0), - rms(0), - complexity(benchmark::oNone) {} - - double coef; - double rms; - benchmark::BigO complexity; + LeastSq() : + coef(0), + rms(0), + complexity(benchmark::oNone) {} + + double coef; + double rms; + benchmark::BigO complexity; }; // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -- cgit v1.2.3 From 43ef17441cc8767f5523031878a2f43ab1d7790b Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 20:50:35 +0200 Subject: refactor names --- src/reporter.cc | 34 +++++++++++++++++----------------- test/complexity_test.cc | 8 ++++---- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/reporter.cc b/src/reporter.cc index 0e1c581..0c05ab0 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -84,48 +84,48 @@ void BenchmarkReporter::ComputeBigO( Run* big_o, Run* rms) { CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; // Accumulators. - std::vector N; - std::vector RealTime; - std::vector CpuTime; + std::vector n; + std::vector real_time; + std::vector cpu_time; // Populate the accumulators. for (const Run& run : reports) { - N.push_back(run.arg1); - RealTime.push_back(run.real_accumulated_time/run.iterations); - CpuTime.push_back(run.cpu_accumulated_time/run.iterations); + n.push_back(run.arg1); + real_time.push_back(run.real_accumulated_time/run.iterations); + cpu_time.push_back(run.cpu_accumulated_time/run.iterations); } - LeastSq resultCpu = MinimalLeastSq(N, CpuTime, reports[0].complexity); + LeastSq result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - // resultCpu.complexity is passed as parameter to resultReal because in case + // result_cpu.complexity is passed as parameter to result_real because in case // reports[0].complexity is oAuto, the noise on the measured data could make // the best fit function of Cpu and Real differ. In order to solve this, we take // the best fitting function for the Cpu, and apply it to Real data. - LeastSq resultReal = MinimalLeastSq(N, RealTime, resultCpu.complexity); + LeastSq result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. big_o->benchmark_name = benchmark_name + "_BigO"; big_o->iterations = 0; - big_o->real_accumulated_time = resultReal.coef; - big_o->cpu_accumulated_time = resultCpu.coef; + big_o->real_accumulated_time = result_real.coef; + big_o->cpu_accumulated_time = result_cpu.coef; big_o->report_big_o = true; - big_o->complexity = resultCpu.complexity; + big_o->complexity = result_cpu.complexity; double multiplier; - const char* timeLabel; - std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(reports[0].time_unit); + const char* time_label; + std::tie(time_label, multiplier) = GetTimeUnitAndMultiplier(reports[0].time_unit); // Only add label to mean/stddev if it is same for all runs big_o->report_label = reports[0].report_label; rms->benchmark_name = benchmark_name + "_RMS"; rms->report_label = big_o->report_label; rms->iterations = 0; - rms->real_accumulated_time = resultReal.rms / multiplier; - rms->cpu_accumulated_time = resultCpu.rms / multiplier; + rms->real_accumulated_time = result_real.rms / multiplier; + rms->cpu_accumulated_time = result_cpu.rms / multiplier; rms->report_rms = true; - rms->complexity = resultCpu.complexity; + rms->complexity = result_cpu.complexity; } std::string BenchmarkReporter::GetBigO(BigO complexity) { diff --git a/test/complexity_test.cc b/test/complexity_test.cc index e81742e..e169ad9 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -31,9 +31,9 @@ BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); static void BM_Complexity_O_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); - const int itemNotInVector = state.range_x()*2; // Test worst case scenario (item not in vector) + const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector) while (state.KeepRunning()) { - benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector)); + benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); } } BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); @@ -71,9 +71,9 @@ BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark:: static void BM_Complexity_O_log_N(benchmark::State& state) { auto m = ConstructRandomMap(state.range_x()); - const int itemNotInVector = state.range_x()*2; // Test worst case scenario (item not in vector) + const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector) while (state.KeepRunning()) { - benchmark::DoNotOptimize(m.find(itemNotInVector)); + benchmark::DoNotOptimize(m.find(item_not_in_vector)); } } BENCHMARK(BM_Complexity_O_log_N) -- cgit v1.2.3 From f434ce3fb650d40db186780ea3506269d5035ffd Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 12:59:12 -0600 Subject: Add myself to the CONTRIBUTORS file --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 67ecb28..dc311af 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -28,6 +28,7 @@ Chris Kennelly Christopher Seymour David Coeurjolly Dominic Hamon +Eric Fiselier Eugene Zhuk Evgeny Safronov Felix Homann -- cgit v1.2.3 From 266b3bd635a37b28d6e92125c615d3e17f5022ea Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 22:09:55 +0200 Subject: changed color and remove iterations --- src/console_reporter.cc | 5 +++-- src/csv_reporter.cc | 14 ++++++++++++-- src/json_reporter.cc | 17 ++++++++++------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 09b91c2..242a94f 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -112,7 +112,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(result.time_unit); - ColorPrintf(COLOR_GREEN, "%-*s ", + ColorPrintf((result.report_big_o ||result.report_rms) ? COLOR_BLUE : COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if(result.report_big_o) { @@ -146,7 +146,8 @@ void ConsoleReporter::PrintRunData(const Run& result) { timeLabel); } - ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); + if(!result.report_big_o && !result.report_rms) + ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); if (!rate.empty()) { ColorPrintf(COLOR_DEFAULT, " %*s", 13, rate.c_str()); diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index df662e4..9bfd66b 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -99,10 +99,20 @@ void CSVReporter::PrintRunData(const Run & run) { ReplaceAll(&name, "\"", "\"\""); std::cout << "\"" << name << "\","; - std::cout << run.iterations << ","; + // Do not print iteration on bigO and RMS report + if(!run.report_big_o && !run.report_rms) + std::cout << run.iterations << ","; + else + std::cout << ","; + std::cout << real_time << ","; std::cout << cpu_time << ","; - std::cout << timeLabel << ","; + + // Do not print timeLabel on RMS report + if(!run.report_rms) + std::cout << timeLabel << ","; + else + std::cout << ","; if (run.bytes_per_second > 0.0) { std::cout << run.bytes_per_second; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index bfa85e4..c9d9cf1 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -162,17 +162,20 @@ void JSONReporter::PrintRunData(Run const& run) { out << indent << FormatKV("name", run.benchmark_name) << ",\n"; - out << indent - << FormatKV("iterations", run.iterations) - << ",\n"; + if(!run.report_big_o && !run.report_rms) { + out << indent + << FormatKV("iterations", run.iterations) + << ",\n"; + } out << indent << FormatKV("real_time", RoundDouble(real_time)) << ",\n"; out << indent - << FormatKV("cpu_time", RoundDouble(cpu_time)) - << ",\n"; - out << indent - << FormatKV("time_unit", timeLabel); + << FormatKV("cpu_time", RoundDouble(cpu_time)); + if(!run.report_rms) { + out << ",\n" << indent + << FormatKV("time_unit", timeLabel); + } if (run.bytes_per_second > 0.0) { out << ",\n" << indent << FormatKV("bytes_per_second", RoundDouble(run.bytes_per_second)); -- cgit v1.2.3 From 0c23d2852f58dd8e264f67045fd6e454cf481c5c Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 22:31:40 +0200 Subject: extracted BigO and GetBigO in own file --- include/benchmark/benchmark_api.h | 15 +------------- include/benchmark/complexity.h | 42 +++++++++++++++++++++++++++++++++++++++ include/benchmark/reporter.h | 1 - src/reporter.cc | 19 ------------------ 4 files changed, 43 insertions(+), 34 deletions(-) create mode 100644 include/benchmark/complexity.h diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 385cbbc..b3ebb27 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -154,6 +154,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include "macros.h" +#include "complexity.h" namespace benchmark { class BenchmarkReporter; @@ -231,20 +232,6 @@ enum TimeUnit { kMillisecond }; -// BigO is passed to a benchmark in order to specify the asymptotic computational -// complexity for the benchmark. In case oAuto is selected, complexity will be -// calculated automatically to the best fit. -enum BigO { - oNone, - o1, - oN, - oNSquared, - oNCubed, - oLogN, - oNLogN, - oAuto -}; - // State is passed to a running Benchmark and contains state for the // benchmark to use. class State { diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h new file mode 100644 index 0000000..6939218 --- /dev/null +++ b/include/benchmark/complexity.h @@ -0,0 +1,42 @@ +#ifndef COMPLEXITY_H_ +#define COMPLEXITY_H_ + +#include + +namespace benchmark { + +// BigO is passed to a benchmark in order to specify the asymptotic computational +// complexity for the benchmark. In case oAuto is selected, complexity will be +// calculated automatically to the best fit. +enum BigO { + oNone, + o1, + oN, + oNSquared, + oNCubed, + oLogN, + oNLogN, + oAuto +}; + +inline std::string GetBigO(BigO complexity) { + switch (complexity) { + case oN: + return "* N"; + case oNSquared: + return "* N**2"; + case oNCubed: + return "* N**3"; + case oLogN: + return "* lgN"; + case oNLogN: + return "* NlgN"; + case o1: + return "* 1"; + default: + return ""; + } +} + +} // end namespace benchmark +#endif // COMPLEXITY_H_ \ No newline at end of file diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 5e0a552..2e164db 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -108,7 +108,6 @@ protected: static void ComputeStats(const std::vector & reports, Run* mean, Run* stddev); static void ComputeBigO(const std::vector & reports, Run* bigO, Run* rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); - static std::string GetBigO(BigO complexity); }; // Simple reporter that outputs benchmark data to the console. This is the diff --git a/src/reporter.cc b/src/reporter.cc index 0c05ab0..f928d69 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -128,25 +128,6 @@ void BenchmarkReporter::ComputeBigO( rms->complexity = result_cpu.complexity; } -std::string BenchmarkReporter::GetBigO(BigO complexity) { - switch (complexity) { - case oN: - return "* N"; - case oNSquared: - return "* N**2"; - case oNCubed: - return "* N**3"; - case oLogN: - return "* lgN"; - case oNLogN: - return "* NlgN"; - case o1: - return "* 1"; - default: - return ""; - } -} - TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) { switch (unit) { case kMillisecond: -- cgit v1.2.3 From 855786acf518db0162779f8196d930820e4f6b8c Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 23 May 2016 22:49:16 +0200 Subject: added end of file carriage return --- include/benchmark/complexity.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h index 6939218..82dba82 100644 --- a/include/benchmark/complexity.h +++ b/include/benchmark/complexity.h @@ -39,4 +39,4 @@ inline std::string GetBigO(BigO complexity) { } } // end namespace benchmark -#endif // COMPLEXITY_H_ \ No newline at end of file +#endif // COMPLEXITY_H_ -- cgit v1.2.3 From 69d1a524079e07508b4af08cbd1e83576387b468 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 15:05:55 -0600 Subject: Add checks that Timing functions are not called outside of the benchmark. Fixes #204 --- include/benchmark/benchmark_api.h | 7 +++++-- src/benchmark.cc | 4 +++- src/check.h | 20 ++++++++++++++---- test/CMakeLists.txt | 3 +++ test/diagnostics_test.cc | 43 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 test/diagnostics_test.cc diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 2ded481..3a8f252 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -242,15 +242,17 @@ public: // returned false. bool KeepRunning() { if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { - ResumeTiming(); + assert(!finished_); started_ = true; + ResumeTiming(); } bool const res = total_iterations_++ < max_iterations; if (BENCHMARK_BUILTIN_EXPECT(!res, false)) { - assert(started_); + assert(started_ && !finished_); PauseTiming(); // Total iterations now is one greater than max iterations. Fix this. total_iterations_ = max_iterations; + finished_ = true; } return res; } @@ -371,6 +373,7 @@ public: private: bool started_; + bool finished_; size_t total_iterations_; bool has_range_x_; diff --git a/src/benchmark.cc b/src/benchmark.cc index 3d12d28..8237a45 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -815,7 +815,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, int thread_i, int n_threads) - : started_(false), total_iterations_(0), + : started_(false), finished_(false), total_iterations_(0), has_range_x_(has_x), range_x_(x), has_range_y_(has_y), range_y_(y), bytes_processed_(0), items_processed_(0), @@ -830,11 +830,13 @@ State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, void State::PauseTiming() { // Add in time accumulated so far CHECK(running_benchmark); + CHECK(started_ && !finished_); timer_manager->StopTimer(); } void State::ResumeTiming() { CHECK(running_benchmark); + CHECK(started_ && !finished_); timer_manager->StartTimer(); } diff --git a/src/check.h b/src/check.h index d2c1fda..72557ac 100644 --- a/src/check.h +++ b/src/check.h @@ -10,6 +10,18 @@ namespace benchmark { namespace internal { +typedef void(AbortHandlerT)(); + +inline AbortHandlerT*& get_abort_handler() { + static AbortHandlerT* handler = &std::abort; + return handler; +} + +BENCHMARK_NORETURN inline void abort_handler() { + get_abort_handler()(); + std::abort(); // fallback to enforce noreturn +} + // CheckHandler is the class constructed by failing CHECK macros. CheckHandler // will log information about the failures and abort when it is destructed. class CheckHandler { @@ -25,13 +37,13 @@ public: return log_; } - BENCHMARK_NORETURN ~CheckHandler() { + BENCHMARK_NORETURN ~CheckHandler() noexcept(false) { log_ << std::endl; - std::abort(); + abort_handler(); } - CheckHandler & operator=(const CheckHandler&) = delete; - CheckHandler(const CheckHandler&) = delete; + CheckHandler & operator=(const CheckHandler&) = delete; + CheckHandler(const CheckHandler&) = delete; CheckHandler() = delete; private: std::ostream& log_; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a10a53a..e83cd1a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -36,6 +36,9 @@ add_test(options_benchmarks options_test --benchmark_min_time=0.01) compile_benchmark_test(basic_test) add_test(basic_benchmark basic_test --benchmark_min_time=0.01) +compile_benchmark_test(diagnostics_test) +add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01) + compile_benchmark_test(fixture_test) add_test(fixture_test fixture_test --benchmark_min_time=0.01) diff --git a/test/diagnostics_test.cc b/test/diagnostics_test.cc new file mode 100644 index 0000000..413d30d --- /dev/null +++ b/test/diagnostics_test.cc @@ -0,0 +1,43 @@ + +#include "benchmark/benchmark_api.h" +#include "../src/check.h" + +void test_handler() { + throw std::logic_error(""); +} + +void try_invalid_pause_resume(benchmark::State& state) { +#ifndef NDEBUG + try { + state.PauseTiming(); + std::abort(); + } catch (std::logic_error const&) {} + try { + state.ResumeTiming(); + std::abort(); + } catch (std::logic_error const&) {} +#else + (void)state; // avoid unused warning +#endif +} + +void BM_diagnostic_test(benchmark::State& state) { + static bool called_once = false; + + if (called_once == false) try_invalid_pause_resume(state); + + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + + if (called_once == false) try_invalid_pause_resume(state); + + called_once = true; +} +BENCHMARK(BM_diagnostic_test); + +int main(int argc, char** argv) { + benchmark::internal::get_abort_handler() = &test_handler; + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} \ No newline at end of file -- cgit v1.2.3 From 751e07d42da22f8ab554394cd3e242fc2ac80002 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 15:07:54 -0600 Subject: add newline --- test/diagnostics_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diagnostics_test.cc b/test/diagnostics_test.cc index 413d30d..6c4aac5 100644 --- a/test/diagnostics_test.cc +++ b/test/diagnostics_test.cc @@ -40,4 +40,4 @@ int main(int argc, char** argv) { benchmark::internal::get_abort_handler() = &test_handler; benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); -} \ No newline at end of file +} -- cgit v1.2.3 From 481e06e65acc2d82e8f3a02fba2fd226de56b3ad Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 15:27:43 -0600 Subject: Address review comments --- include/benchmark/benchmark_api.h | 3 ++- src/check.h | 8 ++++---- test/diagnostics_test.cc | 12 ++++++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 3a8f252..4367265 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -235,7 +235,8 @@ enum TimeUnit { // benchmark to use. class State { public: - State(size_t max_iters, bool has_x, int x, bool has_y, int y, int thread_i, int n_threads); + State(size_t max_iters, bool has_x, int x, bool has_y, int y, + int thread_i, int n_threads); // Returns true iff the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has diff --git a/src/check.h b/src/check.h index 72557ac..cb49c49 100644 --- a/src/check.h +++ b/src/check.h @@ -12,13 +12,13 @@ namespace internal { typedef void(AbortHandlerT)(); -inline AbortHandlerT*& get_abort_handler() { +inline AbortHandlerT*& GetAbortHandler() { static AbortHandlerT* handler = &std::abort; return handler; } -BENCHMARK_NORETURN inline void abort_handler() { - get_abort_handler()(); +BENCHMARK_NORETURN inline void CallAbortHandler() { + GetAbortHandler()(); std::abort(); // fallback to enforce noreturn } @@ -39,7 +39,7 @@ public: BENCHMARK_NORETURN ~CheckHandler() noexcept(false) { log_ << std::endl; - abort_handler(); + CallAbortHandler(); } CheckHandler & operator=(const CheckHandler&) = delete; diff --git a/test/diagnostics_test.cc b/test/diagnostics_test.cc index 6c4aac5..005abdc 100644 --- a/test/diagnostics_test.cc +++ b/test/diagnostics_test.cc @@ -1,8 +1,16 @@ +// Testing: +// State::PauseTiming() +// State::ResumeTiming() +// Test that CHECK's within these function diagnose when they are called +// outside of the KeepRunning() loop. +// +// NOTE: Users should NOT include or use src/check.h. This is only done in +// order to test library internals. #include "benchmark/benchmark_api.h" #include "../src/check.h" -void test_handler() { +void TestHandler() { throw std::logic_error(""); } @@ -37,7 +45,7 @@ void BM_diagnostic_test(benchmark::State& state) { BENCHMARK(BM_diagnostic_test); int main(int argc, char** argv) { - benchmark::internal::get_abort_handler() = &test_handler; + benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); } -- cgit v1.2.3 From 029f37446daeef98db8dc3b26165405a1df2cc80 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 15:38:30 -0600 Subject: Allow test to compile with -fno-exceptions --- test/diagnostics_test.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/diagnostics_test.cc b/test/diagnostics_test.cc index 005abdc..60fa3b1 100644 --- a/test/diagnostics_test.cc +++ b/test/diagnostics_test.cc @@ -9,13 +9,23 @@ #include "benchmark/benchmark_api.h" #include "../src/check.h" +#include +#include + +#if defined(__GNUC__) && !defined(__EXCEPTIONS) +#define TEST_HAS_NO_EXCEPTIONS +#endif void TestHandler() { +#ifndef TEST_HAS_NO_EXCEPTIONS throw std::logic_error(""); +#else + std::abort(); +#endif } void try_invalid_pause_resume(benchmark::State& state) { -#ifndef NDEBUG +#if !defined(NDEBUG) && !defined(TEST_HAS_NO_EXCEPTIONS) try { state.PauseTiming(); std::abort(); -- cgit v1.2.3 From 43017f8b1510a50855e6b0ea145a534d3d754068 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 19:24:56 -0600 Subject: Add SkipWithError(...) function. --- include/benchmark/benchmark_api.h | 19 ++++- include/benchmark/reporter.h | 4 + src/benchmark.cc | 100 ++++++++++++++++------- src/check.h | 20 ++++- src/console_reporter.cc | 18 ++++- src/json_reporter.cc | 8 ++ src/reporter.cc | 2 + test/CMakeLists.txt | 6 ++ test/diagnostics_test.cc | 61 +++++++++++++++ test/skip_with_error_test.cc | 161 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 359 insertions(+), 40 deletions(-) create mode 100644 test/diagnostics_test.cc create mode 100644 test/skip_with_error_test.cc diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 2ded481..654986d 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -235,22 +235,27 @@ enum TimeUnit { // benchmark to use. class State { public: - State(size_t max_iters, bool has_x, int x, bool has_y, int y, int thread_i, int n_threads); + State(size_t max_iters, bool has_x, int x, bool has_y, int y, + int thread_i, int n_threads); // Returns true iff the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has // returned false. bool KeepRunning() { if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { - ResumeTiming(); + assert(!finished_); started_ = true; + ResumeTiming(); } bool const res = total_iterations_++ < max_iterations; if (BENCHMARK_BUILTIN_EXPECT(!res, false)) { - assert(started_); - PauseTiming(); + assert(started_ && (!finished_ || error_occurred_)); + if (!error_occurred_) { + PauseTiming(); + } // Total iterations now is one greater than max iterations. Fix this. total_iterations_ = max_iterations; + finished_ = true; } return res; } @@ -283,6 +288,8 @@ public: // within each benchmark iteration, if possible. void ResumeTiming(); + void SkipWithError(const char* msg); + // REQUIRES: called exactly once per iteration of the KeepRunning loop. // Set the manually measured time for this benchmark iteration, which // is used instead of automatically measured time if UseManualTime() was @@ -371,6 +378,7 @@ public: private: bool started_; + bool finished_; size_t total_iterations_; bool has_range_x_; @@ -382,6 +390,9 @@ private: size_t bytes_processed_; size_t items_processed_; +public: + // FIXME: Make this private somehow. + bool error_occurred_; public: // Index of the executing thread. Values from [0, threads). const int thread_index; diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index aaf5fbf..ac222e4 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -42,6 +42,7 @@ class BenchmarkReporter { struct Run { Run() : + error_occurred(false), iterations(1), time_unit(kNanosecond), real_accumulated_time(0), @@ -52,6 +53,9 @@ class BenchmarkReporter { std::string benchmark_name; std::string report_label; // Empty if not set by benchmark. + bool error_occurred; + std::string error_message; + int64_t iterations; TimeUnit time_unit; double real_accumulated_time; diff --git a/src/benchmark.cc b/src/benchmark.cc index 3d12d28..5d8f279 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -112,6 +112,10 @@ std::string* GetReportLabel() { return &label; } +// Global variable so that a benchmark can report an error as a human readable +// string. If error_message is null no error occurred. +static std::atomic error_message = ATOMIC_VAR_INIT(nullptr); + // TODO(ericwf): support MallocCounter. //static benchmark::MallocCounter *benchmark_mc; @@ -126,6 +130,7 @@ class TimerManager { public: TimerManager(int num_threads, Notification* done) : num_threads_(num_threads), + running_threads_(num_threads), done_(done), running_(false), real_time_used_(0), @@ -133,7 +138,8 @@ class TimerManager { manual_time_used_(0), num_finalized_(0), phase_number_(0), - entered_(0) { + entered_(0) + { } // Called by each thread @@ -196,6 +202,16 @@ class TimerManager { } } + void RemoveErroredThread() EXCLUDES(lock_) { + MutexLock ml(lock_); + int last_thread = --running_threads_ == 0; + if (last_thread && running_) { + InternalStop(); + } else if (!last_thread){ + phase_condition_.notify_all(); + } + } + // REQUIRES: timer is not running double real_time_used() EXCLUDES(lock_) { MutexLock l(lock_); @@ -221,6 +237,7 @@ class TimerManager { Mutex lock_; Condition phase_condition_; int num_threads_; + int running_threads_; Notification* done_; bool running_; // Is the timer running @@ -252,22 +269,27 @@ class TimerManager { // entered the barrier. Returns iff this is the last thread to // enter the barrier. bool Barrier(MutexLock& ml) REQUIRES(lock_) { - CHECK_LT(entered_, num_threads_); + CHECK_LT(entered_, running_threads_); entered_++; - if (entered_ < num_threads_) { + if (entered_ < running_threads_) { // Wait for all threads to enter int phase_number_cp = phase_number_; auto cb = [this, phase_number_cp]() { - return this->phase_number_ > phase_number_cp; + return this->phase_number_ > phase_number_cp || + entered_ == running_threads_; // A thread has aborted in error }; - phase_condition_.wait(ml.native_handle(), cb); - return false; // I was not the last one - } else { - // Last thread has reached the barrier - phase_number_++; - entered_ = 0; - return true; + while (true) { + phase_condition_.wait(ml.native_handle(), cb); + if (phase_number_ > phase_number_cp) + return false; + else if (running_threads_ == entered_) + break; + } } + // Last thread has reached the barrier + phase_number_++; + entered_ = 0; + return true; } }; @@ -702,6 +724,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, MutexLock l(GetBenchmarkLock()); GetReportLabel()->clear(); } + error_message = nullptr; Notification done; timer_manager = std::unique_ptr(new TimerManager(b.threads, &done)); @@ -747,40 +770,48 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, MutexLock l(GetBenchmarkLock()); label = *GetReportLabel(); } + const char* error_msg = error_message; const double min_time = !IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time; // If this was the first run, was elapsed time or cpu time large enough? // If this is not the first run, go with the current value of iter. - if ((i > 0) || + if ((i > 0) || (error_msg != nullptr) || (iters >= kMaxIterations) || (seconds >= min_time) || (real_accumulated_time >= 5*min_time)) { - double bytes_per_second = 0; - if (total.bytes_processed > 0 && seconds > 0.0) { - bytes_per_second = (total.bytes_processed / seconds); - } - double items_per_second = 0; - if (total.items_processed > 0 && seconds > 0.0) { - items_per_second = (total.items_processed / seconds); - } // Create report about this benchmark run. BenchmarkReporter::Run report; report.benchmark_name = b.name; + report.error_occurred = error_msg != nullptr; + report.error_message = error_msg != nullptr ? error_msg : ""; report.report_label = label; // Report the total iterations across all threads. report.iterations = static_cast(iters) * b.threads; report.time_unit = b.time_unit; - if (b.use_manual_time) { - report.real_accumulated_time = manual_accumulated_time; - } else { - report.real_accumulated_time = real_accumulated_time; + + if (!report.error_occurred) { + + double bytes_per_second = 0; + if (total.bytes_processed > 0 && seconds > 0.0) { + bytes_per_second = (total.bytes_processed / seconds); + } + double items_per_second = 0; + if (total.items_processed > 0 && seconds > 0.0) { + items_per_second = (total.items_processed / seconds); + } + + if (b.use_manual_time) { + report.real_accumulated_time = manual_accumulated_time; + } else { + report.real_accumulated_time = real_accumulated_time; + } + report.cpu_accumulated_time = cpu_accumulated_time; + report.bytes_per_second = bytes_per_second; + report.items_per_second = items_per_second; } - report.cpu_accumulated_time = cpu_accumulated_time; - report.bytes_per_second = bytes_per_second; - report.items_per_second = items_per_second; reports.push_back(report); break; } @@ -815,10 +846,11 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, int thread_i, int n_threads) - : started_(false), total_iterations_(0), + : started_(false), finished_(false), total_iterations_(0), has_range_x_(has_x), range_x_(x), has_range_y_(has_y), range_y_(y), bytes_processed_(0), items_processed_(0), + error_occurred_(false), thread_index(thread_i), threads(n_threads), max_iterations(max_iters) @@ -830,14 +862,26 @@ State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, void State::PauseTiming() { // Add in time accumulated so far CHECK(running_benchmark); + CHECK(started_ && !finished_ && !error_occurred_); timer_manager->StopTimer(); } void State::ResumeTiming() { CHECK(running_benchmark); + CHECK(started_ && !finished_); timer_manager->StartTimer(); } +void State::SkipWithError(const char* msg) { + CHECK(msg); + error_occurred_ = true; + const char* expected_no_error_msg = nullptr; + error_message.compare_exchange_weak(expected_no_error_msg, msg); + started_ = finished_ = true; + total_iterations_ = max_iterations; + timer_manager->RemoveErroredThread(); +} + void State::SetIterationTime(double seconds) { CHECK(running_benchmark); diff --git a/src/check.h b/src/check.h index d2c1fda..cb49c49 100644 --- a/src/check.h +++ b/src/check.h @@ -10,6 +10,18 @@ namespace benchmark { namespace internal { +typedef void(AbortHandlerT)(); + +inline AbortHandlerT*& GetAbortHandler() { + static AbortHandlerT* handler = &std::abort; + return handler; +} + +BENCHMARK_NORETURN inline void CallAbortHandler() { + GetAbortHandler()(); + std::abort(); // fallback to enforce noreturn +} + // CheckHandler is the class constructed by failing CHECK macros. CheckHandler // will log information about the failures and abort when it is destructed. class CheckHandler { @@ -25,13 +37,13 @@ public: return log_; } - BENCHMARK_NORETURN ~CheckHandler() { + BENCHMARK_NORETURN ~CheckHandler() noexcept(false) { log_ << std::endl; - std::abort(); + CallAbortHandler(); } - CheckHandler & operator=(const CheckHandler&) = delete; - CheckHandler(const CheckHandler&) = delete; + CheckHandler & operator=(const CheckHandler&) = delete; + CheckHandler(const CheckHandler&) = delete; CheckHandler() = delete; private: std::ostream& log_; diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 56bd3ce..786d030 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -65,7 +66,11 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { PrintRunData(run); } - if (reports.size() < 2) { + auto error_count = std::count_if( + reports.begin(), reports.end(), + [](Run const& run) {return run.error_occurred;}); + + if (reports.size() - error_count < 2) { // We don't report aggregated data if there was a single run. return; } @@ -80,6 +85,14 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { } void ConsoleReporter::PrintRunData(const Run& result) { + ColorPrintf(COLOR_GREEN, "%-*s ", + name_field_width_, result.benchmark_name.c_str()); + + if (result.error_occurred) { + ColorPrintf(COLOR_RED, "ERROR OCCURRED: \'%s\'", result.error_message.c_str()); + ColorPrintf(COLOR_DEFAULT, "\n"); + return; + } // Format bytes per second std::string rate; if (result.bytes_per_second > 0) { @@ -97,9 +110,6 @@ void ConsoleReporter::PrintRunData(const Run& result) { const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(result.time_unit); - ColorPrintf(COLOR_GREEN, "%-*s ", - name_field_width_, result.benchmark_name.c_str()); - if (result.iterations == 0) { ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 7ed141f..84e2e28 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -137,6 +137,14 @@ void JSONReporter::PrintRunData(Run const& run) { out << indent << FormatKV("name", run.benchmark_name) << ",\n"; + if (run.error_occurred) { + out << indent + << FormatKV("error_occurred", run.error_occurred) + << ",\n"; + out << indent + << FormatKV("error_message", run.error_message) + << ",\n"; + } out << indent << FormatKV("iterations", run.iterations) << ",\n"; diff --git a/src/reporter.cc b/src/reporter.cc index 036546e..a23c756 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -39,6 +39,8 @@ void BenchmarkReporter::ComputeStats( for (Run const& run : reports) { CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); CHECK_EQ(run_iterations, run.iterations); + if (run.error_occurred) + continue; real_accumulated_time_stat += Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); cpu_accumulated_time_stat += diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a10a53a..efbe7bb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -36,6 +36,12 @@ add_test(options_benchmarks options_test --benchmark_min_time=0.01) compile_benchmark_test(basic_test) add_test(basic_benchmark basic_test --benchmark_min_time=0.01) +compile_benchmark_test(diagnostics_test) +add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01) + +compile_benchmark_test(skip_with_error_test) +add_test(skip_with_error_test skip_with_error_test --benchmark_min_time=0.01) + compile_benchmark_test(fixture_test) add_test(fixture_test fixture_test --benchmark_min_time=0.01) diff --git a/test/diagnostics_test.cc b/test/diagnostics_test.cc new file mode 100644 index 0000000..60fa3b1 --- /dev/null +++ b/test/diagnostics_test.cc @@ -0,0 +1,61 @@ +// Testing: +// State::PauseTiming() +// State::ResumeTiming() +// Test that CHECK's within these function diagnose when they are called +// outside of the KeepRunning() loop. +// +// NOTE: Users should NOT include or use src/check.h. This is only done in +// order to test library internals. + +#include "benchmark/benchmark_api.h" +#include "../src/check.h" +#include +#include + +#if defined(__GNUC__) && !defined(__EXCEPTIONS) +#define TEST_HAS_NO_EXCEPTIONS +#endif + +void TestHandler() { +#ifndef TEST_HAS_NO_EXCEPTIONS + throw std::logic_error(""); +#else + std::abort(); +#endif +} + +void try_invalid_pause_resume(benchmark::State& state) { +#if !defined(NDEBUG) && !defined(TEST_HAS_NO_EXCEPTIONS) + try { + state.PauseTiming(); + std::abort(); + } catch (std::logic_error const&) {} + try { + state.ResumeTiming(); + std::abort(); + } catch (std::logic_error const&) {} +#else + (void)state; // avoid unused warning +#endif +} + +void BM_diagnostic_test(benchmark::State& state) { + static bool called_once = false; + + if (called_once == false) try_invalid_pause_resume(state); + + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + + if (called_once == false) try_invalid_pause_resume(state); + + called_once = true; +} +BENCHMARK(BM_diagnostic_test); + +int main(int argc, char** argv) { + benchmark::internal::GetAbortHandler() = &TestHandler; + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc new file mode 100644 index 0000000..dafbd64 --- /dev/null +++ b/test/skip_with_error_test.cc @@ -0,0 +1,161 @@ + +#undef NDEBUG +#include "benchmark/benchmark.h" +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include +#include + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + TestReporter() {} + virtual ~TestReporter() {} + + mutable std::vector all_runs_; +}; + +struct TestCase { + std::string name; + bool error_occurred; + std::string error_message; + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + CHECK(name == run.benchmark_name) << "expected " << name << " got " << run.benchmark_name; + CHECK(error_occurred == run.error_occurred); + CHECK(error_message == run.error_message); + if (error_occurred) { + //CHECK(run.iterations == 0); + } else { + CHECK(run.iterations != 0); + } + } +}; + +std::vector ExpectedResults; + +int AddCases(const char* base_name, std::initializer_list const& v) { + for (auto TC : v) { + TC.name = base_name + TC.name; + ExpectedResults.push_back(std::move(TC)); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) \ +int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +} // end namespace + + +void BM_error_before_running(benchmark::State& state) { + state.SkipWithError("error message"); + while (state.KeepRunning()) { + assert(false); + } +} +BENCHMARK(BM_error_before_running); +ADD_CASES("BM_error_before_running", + {{"", true, "error message"}}); + +void BM_error_during_running(benchmark::State& state) { + int first_iter = true; + while (state.KeepRunning()) { + if (state.range_x() == 1 && state.thread_index <= (state.threads / 2)) { + assert(first_iter); + first_iter = false; + state.SkipWithError("error message"); + } else { + state.PauseTiming(); + state.ResumeTiming(); + } + } +} +BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8); +ADD_CASES( + "BM_error_during_running", + {{"/1/threads:1", true, "error message"}, + {"/1/threads:2", true, "error message"}, + {"/1/threads:4", true, "error message"}, + {"/1/threads:8", true, "error message"}, + {"/2/threads:1", false, ""}, + {"/2/threads:2", false, ""}, + {"/2/threads:4", false, ""}, + {"/2/threads:8", false, ""}} +); + +void BM_error_after_running(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + if (state.thread_index <= (state.threads / 2)) + state.SkipWithError("error message"); +} +BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); +ADD_CASES( + "BM_error_after_running", + {{"/threads:1", true, "error message"}, + {"/threads:2", true, "error message"}, + {"/threads:4", true, "error message"}, + {"/threads:8", true, "error message"}} +); + +void BM_error_while_paused(benchmark::State& state) { + bool first_iter = true; + while (state.KeepRunning()) { + if (state.range_x() == 1 && state.thread_index <= (state.threads / 2)) { + assert(first_iter); + first_iter = false; + state.PauseTiming(); + state.SkipWithError("error message"); + } else { + state.PauseTiming(); + state.ResumeTiming(); + } + } +} +BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8); +ADD_CASES( + "BM_error_while_paused", + {{"/1/threads:1", true, "error message"}, + {"/1/threads:2", true, "error message"}, + {"/1/threads:4", true, "error message"}, + {"/1/threads:8", true, "error message"}, + {"/2/threads:1", false, ""}, + {"/2/threads:2", false, ""}, + {"/2/threads:4", false, ""}, + {"/2/threads:8", false, ""}} +); + + +int main(int argc, char* argv[]) { + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); + + return 0; +} -- cgit v1.2.3 From 90c9ab1d8e0a44d229dae6b4f5f6355161de761a Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 20:35:09 -0600 Subject: add doc --- README.md | 31 +++++++++++++++++++++++++++++++ include/benchmark/benchmark_api.h | 25 +++++++++++++++++++++---- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 051b301..f9f3bf4 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,37 @@ BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); /* BarTest is now registered */ ``` +## Exiting Benchmarks in Error + +When external influences such as file I/O and network errors occur within +a benchmark the `State::SkipWithError(const char* msg)` function can be used +to skip that run of benchmark and report the error. Note that only future +iterations of the `KeepRunning()` are skipped. Users may explicitly return +to exit the benchmark immediately. + +The `SkipWithError(...)` function may be used at any point within the benchmark, +including before and after the `KeepRunning()` loop. + +For example: + +```c++ +static void BM_test(benchmark::State& state) { + auto resource = GetResource(); + if (!resource.good()) { + state.SkipWithError("Resource is not good!"); + // KeepRunning() loop will not be entered. + } + while (state.KeepRunning()) { + auto data = resource.read_data(); + if (!resource.good()) { + state.SkipWithError("Failed to read data!"); + break; // Needed to skip the rest of the iteration. + } + do_stuff(data); + } +} +``` + ## Output Formats The library supports multiple output formats. Use the `--benchmark_format=` flag to set the format type. `tabular` is diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 654986d..7100bf5 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -260,13 +260,14 @@ public: return res; } - // REQUIRES: timer is running + // REQUIRES: timer is running and 'SkipWithError(...)' has not been called + // in the current thread. // Stop the benchmark timer. If not called, the timer will be // automatically stopped after KeepRunning() returns false for the first time. // // For threaded benchmarks the PauseTiming() function acts // like a barrier. I.e., the ith call by a particular thread to this - // function will block until all threads have made their ith call. + // function will block until all active threads have made their ith call. // The timer will stop when the last thread has called this function. // // NOTE: PauseTiming()/ResumeTiming() are relatively @@ -274,13 +275,14 @@ public: // within each benchmark iteration, if possible. void PauseTiming(); - // REQUIRES: timer is not running + // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called + // in the current thread. // Start the benchmark timer. The timer is NOT running on entrance to the // benchmark function. It begins running after the first call to KeepRunning() // // For threaded benchmarks the ResumeTiming() function acts // like a barrier. I.e., the ith call by a particular thread to this - // function will block until all threads have made their ith call. + // function will block until all active threads have made their ith call. // The timer will start when the last thread has called this function. // // NOTE: PauseTiming()/ResumeTiming() are relatively @@ -288,6 +290,21 @@ public: // within each benchmark iteration, if possible. void ResumeTiming(); + // REQUIRES: 'SkipWithError(...)' has not been called previously in the + // current thread. + // Skip any future iterations of the 'KeepRunning()' loop in the current + // thread and report an error with the specified 'msg'. After this call + // the user may explicitly 'return' from the benchmark. + // + // For threaded benchmarks only the current thread stops executing. If + // multiple threads report an error only the first error message will be used. + // The current thread is no longer considered 'active' thread by + // 'PauseTiming()' and 'ResumingTiming()'. + // + // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit + // the current scope immediately. If the function is called from within + // the 'KeepRunning()' loop the current iteration will finish. It is the users + // responsibility to exit the scope as needed. void SkipWithError(const char* msg); // REQUIRES: called exactly once per iteration of the KeepRunning loop. -- cgit v1.2.3 From c7108df977c40c7794701c369a8cb66135d75e14 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 20:44:10 -0600 Subject: Only compile and run the C++03 test when -std=c++03 is supported. Fixes #164 --- test/CMakeLists.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a10a53a..dd62b64 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -2,10 +2,6 @@ find_package(Threads REQUIRED) -set(CXX03_FLAGS "${CMAKE_CXX_FLAGS}") -string(REPLACE "-std=c++11" "-std=c++03" CXX03_FLAGS "${CXX03_FLAGS}") -string(REPLACE "-std=c++0x" "-std=c++03" CXX03_FLAGS "${CXX03_FLAGS}") - macro(compile_benchmark_test name) add_executable(${name} "${name}.cc") target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT}) @@ -42,10 +38,18 @@ add_test(fixture_test fixture_test --benchmark_min_time=0.01) compile_benchmark_test(map_test) add_test(map_test map_test --benchmark_min_time=0.01) -compile_benchmark_test(cxx03_test) -set_target_properties(cxx03_test - PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}") -add_test(cxx03 cxx03_test --benchmark_min_time=0.01) + +check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) +if (BENCHMARK_HAS_CXX03_FLAG) + set(CXX03_FLAGS "${CMAKE_CXX_FLAGS}") + string(REPLACE "-std=c++11" "-std=c++03" CXX03_FLAGS "${CXX03_FLAGS}") + string(REPLACE "-std=c++0x" "-std=c++03" CXX03_FLAGS "${CXX03_FLAGS}") + + compile_benchmark_test(cxx03_test) + set_target_properties(cxx03_test + PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}") + add_test(cxx03 cxx03_test --benchmark_min_time=0.01) +endif() # Add the coverage command(s) if(CMAKE_BUILD_TYPE) -- cgit v1.2.3 From 9fcdd6fc25fcb232038355c6103eb6f06bcd5dd5 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 20:51:15 -0600 Subject: Prevent re-execution of slow feature tests. --- cmake/CXXFeatureCheck.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index 23ee8ac..3059024 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -21,12 +21,15 @@ function(cxx_feature_check FILE) string(TOLOWER ${FILE} FILE) string(TOUPPER ${FILE} VAR) string(TOUPPER "HAVE_${VAR}" FEATURE) + if (DEFINED HAVE_${VAR}) + return() + endif() message("-- Performing Test ${FEATURE}") try_run(RUN_${FEATURE} COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp) if(RUN_${FEATURE} EQUAL 0) message("-- Performing Test ${FEATURE} -- success") - set(HAVE_${VAR} 1 PARENT_SCOPE) + set(HAVE_${VAR} 1 CACHE INTERNAL "Feature test for ${FILE}" PARENT_SCOPE) add_definitions(-DHAVE_${VAR}) else() if(NOT COMPILE_${FEATURE}) -- cgit v1.2.3 From e0de8171c678c0f0c643efa9e18de40ad119122b Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 23 May 2016 23:32:05 -0600 Subject: Change RunSpecifiedBenchmarks to return the number of benchmarks run. Fixes #145 --- include/benchmark/benchmark_api.h | 12 ++++++++---- src/benchmark.cc | 20 +++++++++++--------- test/filter_test.cc | 8 ++++---- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 2ded481..81f7b57 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -160,10 +160,14 @@ class BenchmarkReporter; void Initialize(int* argc, char** argv); -// Otherwise, run all benchmarks specified by the --benchmark_filter flag, -// and exit after running the benchmarks. -void RunSpecifiedBenchmarks(); -void RunSpecifiedBenchmarks(BenchmarkReporter* reporter); +// Run all benchmarks which match the specified --benchmark_filter flag. +// The second overload reports the results using the specified 'reporter'. +// +// RETURNS: The number of benchmarks run, not including repetitions. If +// '--benchmark_list_tests' is specified '0' is returned. +size_t RunSpecifiedBenchmarks(); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter); + // If this routine is called, peak memory allocation past this point in the // benchmark is reported at the end of the benchmark report line. (It is diff --git a/src/benchmark.cc b/src/benchmark.cc index 3d12d28..f78f2a3 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -863,14 +863,14 @@ void PrintBenchmarkList() { } } -void RunMatchingBenchmarks(const std::string& spec, - BenchmarkReporter* reporter) { +size_t RunMatchingBenchmarks(const std::string& spec, + BenchmarkReporter* reporter) { CHECK(reporter != nullptr); - if (spec.empty()) return; + CHECK(!spec.empty()); std::vector benchmarks; auto families = BenchmarkFamilies::GetInstance(); - if (!families->FindBenchmarks(spec, &benchmarks)) return; + if (!families->FindBenchmarks(spec, &benchmarks)) return 0; // Determine the width of the name field using a minimum width of 10. size_t name_field_width = 10; @@ -894,6 +894,7 @@ void RunMatchingBenchmarks(const std::string& spec, RunBenchmark(benchmark, reporter); } } + return benchmarks.size(); } std::unique_ptr GetDefaultReporter() { @@ -913,14 +914,14 @@ std::unique_ptr GetDefaultReporter() { } // end namespace } // end namespace internal -void RunSpecifiedBenchmarks() { - RunSpecifiedBenchmarks(nullptr); +size_t RunSpecifiedBenchmarks() { + return RunSpecifiedBenchmarks(nullptr); } -void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { +size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { if (FLAGS_benchmark_list_tests) { internal::PrintBenchmarkList(); - return; + return 0; } std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") @@ -931,8 +932,9 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { default_reporter = internal::GetDefaultReporter(); reporter = default_reporter.get(); } - internal::RunMatchingBenchmarks(spec, reporter); + size_t num_run = internal::RunMatchingBenchmarks(spec, reporter); reporter->Finalize(); + return num_run; } namespace internal { diff --git a/test/filter_test.cc b/test/filter_test.cc index 2a278ff..3f58242 100644 --- a/test/filter_test.cc +++ b/test/filter_test.cc @@ -72,7 +72,7 @@ int main(int argc, char* argv[]) { benchmark::Initialize(&argc, argv); TestReporter test_reporter; - benchmark::RunSpecifiedBenchmarks(&test_reporter); + const size_t returned_count = benchmark::RunSpecifiedBenchmarks(&test_reporter); if (argc == 2) { // Make sure we ran all of the tests @@ -81,9 +81,9 @@ int main(int argc, char* argv[]) { ss >> expected; const size_t count = test_reporter.GetCount(); - if (count != expected) { - std::cerr << "ERROR: Expected " << expected << " tests to be ran but only " - << count << " completed" << std::endl; + if (count != expected || returned_count != expected) { + std::cerr << "ERROR: Expected " << expected << " tests to be run but returned_count = " + << returned_count << " and reporter_count = " << count << std::endl; return -1; } } -- cgit v1.2.3 From 9341d705a14afed7c4e0f8a22b1e6e8a6259fcc4 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 00:42:11 -0600 Subject: Change --benchmark_list_tests to respect the benchmark filter. This behavior mirrors how GTest handles these two flags. --- include/benchmark/benchmark_api.h | 9 +++++--- src/benchmark.cc | 47 +++++++++++++++------------------------ test/CMakeLists.txt | 1 + test/filter_test.cc | 28 +++++++++++++++++------ 4 files changed, 46 insertions(+), 39 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 81f7b57..fb698e3 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -160,11 +160,14 @@ class BenchmarkReporter; void Initialize(int* argc, char** argv); -// Run all benchmarks which match the specified --benchmark_filter flag. +// Generate a list of benchmarks matching the specified --benchmark_filter flag +// and if --benchmark_list_tests is specified return after printing the name +// of each matching benchmark. Otherwise run each matching benchmark and +// report the results. +// // The second overload reports the results using the specified 'reporter'. // -// RETURNS: The number of benchmarks run, not including repetitions. If -// '--benchmark_list_tests' is specified '0' is returned. +// RETURNS: The number of matching benchmarks. size_t RunSpecifiedBenchmarks(); size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter); diff --git a/src/benchmark.cc b/src/benchmark.cc index f78f2a3..86109dd 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -853,24 +853,9 @@ void State::SetLabel(const char* label) { namespace internal { namespace { -void PrintBenchmarkList() { - std::vector benchmarks; - auto families = BenchmarkFamilies::GetInstance(); - if (!families->FindBenchmarks(".", &benchmarks)) return; - - for (const internal::Benchmark::Instance& benchmark : benchmarks) { - std::cout << benchmark.name << "\n"; - } -} - -size_t RunMatchingBenchmarks(const std::string& spec, - BenchmarkReporter* reporter) { +void RunMatchingBenchmarks(const std::vector& benchmarks, + BenchmarkReporter* reporter) { CHECK(reporter != nullptr); - CHECK(!spec.empty()); - - std::vector benchmarks; - auto families = BenchmarkFamilies::GetInstance(); - if (!families->FindBenchmarks(spec, &benchmarks)) return 0; // Determine the width of the name field using a minimum width of 10. size_t name_field_width = 10; @@ -894,7 +879,6 @@ size_t RunMatchingBenchmarks(const std::string& spec, RunBenchmark(benchmark, reporter); } } - return benchmarks.size(); } std::unique_ptr GetDefaultReporter() { @@ -919,22 +903,27 @@ size_t RunSpecifiedBenchmarks() { } size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { - if (FLAGS_benchmark_list_tests) { - internal::PrintBenchmarkList(); - return 0; - } std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") spec = "."; // Regexp that matches all benchmarks - std::unique_ptr default_reporter; - if (!reporter) { - default_reporter = internal::GetDefaultReporter(); - reporter = default_reporter.get(); + std::vector benchmarks; + auto families = internal::BenchmarkFamilies::GetInstance(); + if (!families->FindBenchmarks(spec, &benchmarks)) return 0; + + if (FLAGS_benchmark_list_tests) { + for (auto const& benchmark : benchmarks) + std::cout << benchmark.name << "\n"; + } else { + std::unique_ptr default_reporter; + if (!reporter) { + default_reporter = internal::GetDefaultReporter(); + reporter = default_reporter.get(); + } + internal::RunMatchingBenchmarks(benchmarks, reporter); + reporter->Finalize(); } - size_t num_run = internal::RunMatchingBenchmarks(spec, reporter); - reporter->Finalize(); - return num_run; + return benchmarks.size(); } namespace internal { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dd62b64..419865d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,6 +14,7 @@ add_test(benchmark benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) add_test(${name} filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) + add_test(${name}_list_only filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) add_filter_test(filter_simple "Foo" 3) diff --git a/test/filter_test.cc b/test/filter_test.cc index 3f58242..0ba4071 100644 --- a/test/filter_test.cc +++ b/test/filter_test.cc @@ -68,7 +68,11 @@ BENCHMARK(BM_FooBa); -int main(int argc, char* argv[]) { +int main(int argc, char** argv) { + bool list_only = false; + for (int i=0; i < argc; ++i) + list_only |= std::string(argv[i]).find("--benchmark_list_tests") != std::string::npos; + benchmark::Initialize(&argc, argv); TestReporter test_reporter; @@ -77,15 +81,25 @@ int main(int argc, char* argv[]) { if (argc == 2) { // Make sure we ran all of the tests std::stringstream ss(argv[1]); - size_t expected; - ss >> expected; + size_t expected_return; + ss >> expected_return; - const size_t count = test_reporter.GetCount(); - if (count != expected || returned_count != expected) { - std::cerr << "ERROR: Expected " << expected << " tests to be run but returned_count = " - << returned_count << " and reporter_count = " << count << std::endl; + if (returned_count != expected_return) { + std::cerr << "ERROR: Expected " << expected_return + << " tests to match the filter but returned_count = " + << returned_count << std::endl; + return -1; + } + + const size_t expected_reports = list_only ? 0 : expected_return; + const size_t reports_count = test_reporter.GetCount(); + if (reports_count != expected_reports) { + std::cerr << "ERROR: Expected " << expected_reports + << " tests to be run but reported_count = " << reports_count + << std::endl; return -1; } } + return 0; } -- cgit v1.2.3 From 36a9ae197f220df65ef7ed1a21015a8fb4ef1ece Mon Sep 17 00:00:00 2001 From: Ismael Date: Tue, 24 May 2016 19:56:49 +0200 Subject: added SetComplexityN --- include/benchmark/benchmark_api.h | 17 ++++++++++++++++- include/benchmark/reporter.h | 6 ++---- src/benchmark.cc | 10 ++++++---- src/reporter.cc | 2 +- test/complexity_test.cc | 6 ++++++ 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index b3ebb27..5f1ee20 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -312,6 +312,19 @@ public: return bytes_processed_; } + // If this routine is called with complexity_n > 0 and complexity report is requested for the + // family benchmark, then current benchmark will be part of the computation and complexity_n will + // represent the length of N. + BENCHMARK_ALWAYS_INLINE + void SetComplexityN(size_t complexity_n) { + complexity_n_ = complexity_n; + } + + BENCHMARK_ALWAYS_INLINE + size_t complexity_n() { + return complexity_n_; + } + // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently // executing benchmark. It is typically called at the end of a processing @@ -383,6 +396,8 @@ private: size_t bytes_processed_; size_t items_processed_; + size_t complexity_n_; + public: // Index of the executing thread. Values from [0, threads). const int thread_index; @@ -466,7 +481,7 @@ public: // to control how many iterations are run, and in the printing of items/second // or MB/second values. Benchmark* UseManualTime(); - + // Set the asymptotic computational complexity for the benchmark. If called // the asymptotic computational complexity will be shown on the output. Benchmark* Complexity(BigO complexity); diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 2e164db..8656402 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -50,8 +50,7 @@ class BenchmarkReporter { items_per_second(0), max_heapbytes_used(0), complexity(oNone), - arg1(0), - arg2(0), + complexity_n(0), report_big_o(false), report_rms(false) {} @@ -71,8 +70,7 @@ class BenchmarkReporter { // Keep track of arguments to compute asymptotic complexity BigO complexity; - int arg1; - int arg2; + int complexity_n; // Inform print function whether the current run is a complexity report bool report_big_o; diff --git a/src/benchmark.cc b/src/benchmark.cc index 2245389..03f524b 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -116,9 +116,10 @@ std::string* GetReportLabel() { //static benchmark::MallocCounter *benchmark_mc; struct ThreadStats { - ThreadStats() : bytes_processed(0), items_processed(0) {} + ThreadStats() : bytes_processed(0), items_processed(0), complexity_n(0) {} int64_t bytes_processed; int64_t items_processed; + size_t complexity_n; }; // Timer management class @@ -693,6 +694,7 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, MutexLock l(GetBenchmarkLock()); total->bytes_processed += st.bytes_processed(); total->items_processed += st.items_processed(); + total->complexity_n += st.complexity_n(); } timer_manager->Finalize(); @@ -798,8 +800,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.cpu_accumulated_time = cpu_accumulated_time; report.bytes_per_second = bytes_per_second; report.items_per_second = items_per_second; - report.arg1 = b.arg1; - report.arg2 = b.arg2; + report.complexity_n = total.complexity_n; report.complexity = b.complexity; reports.push_back(report); @@ -851,7 +852,8 @@ State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, bytes_processed_(0), items_processed_(0), thread_index(thread_i), threads(n_threads), - max_iterations(max_iters) + max_iterations(max_iters), + complexity_n_(0) { CHECK(max_iterations != 0) << "At least one iteration must be run"; CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; diff --git a/src/reporter.cc b/src/reporter.cc index f928d69..544df87 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -90,7 +90,7 @@ void BenchmarkReporter::ComputeBigO( // Populate the accumulators. for (const Run& run : reports) { - n.push_back(run.arg1); + n.push_back(run.complexity_n); real_time.push_back(run.real_accumulated_time/run.iterations); cpu_time.push_back(run.cpu_accumulated_time/run.iterations); } diff --git a/test/complexity_test.cc b/test/complexity_test.cc index e169ad9..e454ee4 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -26,6 +26,7 @@ std::map ConstructRandomMap(int size) { void BM_Complexity_O1(benchmark::State& state) { while (state.KeepRunning()) { } + state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); @@ -35,6 +36,7 @@ static void BM_Complexity_O_N(benchmark::State& state) { while (state.KeepRunning()) { benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); } + state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); @@ -42,6 +44,7 @@ BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Com static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); std::string s2(state.range_x(), '-'); + state.SetComplexityN(state.range_x()); while (state.KeepRunning()) for(char& c1 : s1) { for(char& c2 : s2) { @@ -56,6 +59,7 @@ static void BM_Complexity_O_N_Cubed(benchmark::State& state) { std::string s1(state.range_x(), '-'); std::string s2(state.range_x(), '-'); std::string s3(state.range_x(), '-'); + state.SetComplexityN(state.range_x()); while (state.KeepRunning()) for(char& c1 : s1) { for(char& c2 : s2) { @@ -75,6 +79,7 @@ static void BM_Complexity_O_log_N(benchmark::State& state) { while (state.KeepRunning()) { benchmark::DoNotOptimize(m.find(item_not_in_vector)); } + state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oLogN); @@ -84,6 +89,7 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { while (state.KeepRunning()) { std::sort(v.begin(), v.end()); } + state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); -- cgit v1.2.3 From e5cf020d9730d24391e1cd1dfb37aef2c163d82c Mon Sep 17 00:00:00 2001 From: Ismael Date: Tue, 24 May 2016 20:06:54 +0200 Subject: fixed warning --- src/benchmark.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 03f524b..c515e8d 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -850,10 +850,10 @@ State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, has_range_x_(has_x), range_x_(x), has_range_y_(has_y), range_y_(y), bytes_processed_(0), items_processed_(0), + complexity_n_(0), thread_index(thread_i), threads(n_threads), - max_iterations(max_iters), - complexity_n_(0) + max_iterations(max_iters) { CHECK(max_iterations != 0) << "At least one iteration must be run"; CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; -- cgit v1.2.3 From 5e10e120db2ddf36d75e910ec2c77adebbf7543f Mon Sep 17 00:00:00 2001 From: Ismael Date: Tue, 24 May 2016 20:26:21 +0200 Subject: fixed overshadow --- include/benchmark/benchmark_api.h | 2 +- src/benchmark.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 5f1ee20..04ec624 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -321,7 +321,7 @@ public: } BENCHMARK_ALWAYS_INLINE - size_t complexity_n() { + size_t complexity_length_n() { return complexity_n_; } diff --git a/src/benchmark.cc b/src/benchmark.cc index c515e8d..e13fa52 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -694,7 +694,7 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, MutexLock l(GetBenchmarkLock()); total->bytes_processed += st.bytes_processed(); total->items_processed += st.items_processed(); - total->complexity_n += st.complexity_n(); + total->complexity_n += st.complexity_length_n(); } timer_manager->Finalize(); -- cgit v1.2.3 From 64d72ee7b27c719be3233b1f1dd691084e07ffde Mon Sep 17 00:00:00 2001 From: Ismael Date: Tue, 24 May 2016 20:35:39 +0200 Subject: changed complexity_n from int to size_t --- include/benchmark/reporter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 8656402..f37ef20 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -70,7 +70,7 @@ class BenchmarkReporter { // Keep track of arguments to compute asymptotic complexity BigO complexity; - int complexity_n; + size_t complexity_n; // Inform print function whether the current run is a complexity report bool report_big_o; -- cgit v1.2.3 From a24ef95e1199c463bb088d321caa1b0591817813 Mon Sep 17 00:00:00 2001 From: Ismael Date: Tue, 24 May 2016 20:48:34 +0200 Subject: adapt complexity_n to leastsq inteface --- include/benchmark/reporter.h | 2 +- src/benchmark.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index f37ef20..a912488 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -70,7 +70,7 @@ class BenchmarkReporter { // Keep track of arguments to compute asymptotic complexity BigO complexity; - size_t complexity_n; + int complexity_n; // Inform print function whether the current run is a complexity report bool report_big_o; diff --git a/src/benchmark.cc b/src/benchmark.cc index e13fa52..84f88ed 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -119,7 +119,7 @@ struct ThreadStats { ThreadStats() : bytes_processed(0), items_processed(0), complexity_n(0) {} int64_t bytes_processed; int64_t items_processed; - size_t complexity_n; + int complexity_n; }; // Timer management class -- cgit v1.2.3 From f126852c8fe0cbd4f702d696b88d3ca8c4911499 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 24 May 2016 13:15:16 -0700 Subject: simplify format string for complexity output --- src/console_reporter.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 242a94f..cf78a7f 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -124,11 +124,9 @@ void ConsoleReporter::PrintRunData(const Run& result) { big_o.c_str()); } else if(result.report_rms) { - ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", + ColorPrintf(COLOR_YELLOW, "%10.0f %% %10.0f %% ", result.real_accumulated_time * multiplier * 100, - "%", - result.cpu_accumulated_time * multiplier * 100, - "%"); + result.cpu_accumulated_time * multiplier * 100); } else if (result.iterations == 0) { ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", -- cgit v1.2.3 From 2440b752fd335d00349b6dd77d67e5a6401565fb Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 24 May 2016 13:25:59 -0700 Subject: Formatting updates --- README.md | 15 ++++++++---- include/benchmark/benchmark_api.h | 2 +- include/benchmark/complexity.h | 20 +++++++-------- include/benchmark/reporter.h | 11 +++++---- src/benchmark.cc | 13 +++++----- src/console_reporter.cc | 16 ++++++------ src/csv_reporter.cc | 26 ++++++++++---------- src/json_reporter.cc | 6 ++--- src/minimal_leastsq.cc | 51 ++++++++++++++++++++++++--------------- src/minimal_leastsq.h | 19 +++++++++------ src/reporter.cc | 21 +++++++++------- test/complexity_test.cc | 8 +++--- 12 files changed, 118 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index c989e57..498c4ca 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,9 @@ the specified range and will generate a benchmark for each such argument. BENCHMARK(BM_memcpy)->Range(8, 8<<10); ``` -By default the arguments in the range are generated in multiples of eight and the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the range multiplier is changed to multiples of two. +By default the arguments in the range are generated in multiples of eight and +the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the +range multiplier is changed to multiples of two. ```c++ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); @@ -117,7 +119,9 @@ BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` ### Calculate asymptotic complexity (Big O) -Asymptotic complexity might be calculated for a family of benchmarks. The following code will calculate the coefficient for the high-order term in the running time and the normalized root-mean square error of string comparison. +Asymptotic complexity might be calculated for a family of benchmarks. The +following code will calculate the coefficient for the high-order term in the +running time and the normalized root-mean square error of string comparison. ```c++ static void BM_StringCompare(benchmark::State& state) { @@ -127,14 +131,15 @@ static void BM_StringCompare(benchmark::State& state) { benchmark::DoNotOptimize(s1.compare(s2)); } BENCHMARK(BM_StringCompare) - ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN); + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN); ``` -As shown in the following invocation, asymptotic complexity might also be calculated automatically. +As shown in the following invocation, asymptotic complexity might also be +calculated automatically. ```c++ BENCHMARK(BM_StringCompare) - ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oAuto); + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oAuto); ``` ### Templated benchmarks diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index bf46d97..674b0b5 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -327,7 +327,7 @@ public: // represent the length of N. BENCHMARK_ALWAYS_INLINE void SetComplexityN(size_t complexity_n) { - complexity_n_ = complexity_n; + complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h index 82dba82..93b26de 100644 --- a/include/benchmark/complexity.h +++ b/include/benchmark/complexity.h @@ -9,14 +9,14 @@ namespace benchmark { // complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. enum BigO { - oNone, - o1, - oN, - oNSquared, - oNCubed, - oLogN, - oNLogN, - oAuto + oNone, + o1, + oN, + oNSquared, + oNCubed, + oLogN, + oNLogN, + oAuto }; inline std::string GetBigO(BigO complexity) { @@ -34,9 +34,9 @@ inline std::string GetBigO(BigO complexity) { case o1: return "* 1"; default: - return ""; + return ""; } } - + } // end namespace benchmark #endif // COMPLEXITY_H_ diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index a912488..4a67f17 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -67,11 +67,11 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; - + // Keep track of arguments to compute asymptotic complexity BigO complexity; int complexity_n; - + // Inform print function whether the current run is a complexity report bool report_big_o; bool report_rms; @@ -90,7 +90,7 @@ class BenchmarkReporter { // Note that all the grouped benchmark runs should refer to the same // benchmark, thus have the same name. virtual void ReportRuns(const std::vector& report) = 0; - + // Called once at the last benchmark in a family of benchmarks, gives information // about asymptotic complexity and RMS. // Note that all the benchmark runs in a range should refer to the same benchmark, @@ -103,8 +103,9 @@ class BenchmarkReporter { virtual ~BenchmarkReporter(); protected: - static void ComputeStats(const std::vector & reports, Run* mean, Run* stddev); - static void ComputeBigO(const std::vector & reports, Run* bigO, Run* rms); + static void ComputeStats(const std::vector& reports, + Run* mean, Run* stddev); + static void ComputeBigO(const std::vector& reports, Run* bigO, Run* rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); }; diff --git a/src/benchmark.cc b/src/benchmark.cc index 15274d8..bd9858f 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -702,7 +702,8 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, BenchmarkReporter* br, - std::vector& complexity_reports) EXCLUDES(GetBenchmarkLock()) { + std::vector& complexity_reports) + EXCLUDES(GetBenchmarkLock()) { size_t iters = 1; std::vector reports; @@ -803,10 +804,10 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.complexity_n = total.complexity_n; report.complexity = b.complexity; reports.push_back(report); - - if(report.complexity != oNone) + + if(report.complexity != oNone) complexity_reports.push_back(report); - + break; } @@ -830,12 +831,12 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, } } br->ReportRuns(reports); - + if((b.complexity != oNone) && b.last_benchmark_instance) { br->ReportComplexity(complexity_reports); complexity_reports.clear(); } - + if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); diff --git a/src/console_reporter.cc b/src/console_reporter.cc index cf78a7f..41c00b9 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -84,11 +84,11 @@ void ConsoleReporter::ReportComplexity(const std::vector & complexity_repor // We don't report asymptotic complexity data if there was a single run. return; } - + Run big_o_data; Run rms_data; BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); - + // Output using PrintRun. PrintRunData(big_o_data); PrintRunData(rms_data); @@ -112,7 +112,8 @@ void ConsoleReporter::PrintRunData(const Run& result) { const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(result.time_unit); - ColorPrintf((result.report_big_o ||result.report_rms) ? COLOR_BLUE : COLOR_GREEN, "%-*s ", + ColorPrintf((result.report_big_o ||result.report_rms) ? COLOR_BLUE : + COLOR_GREEN, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if(result.report_big_o) { @@ -122,13 +123,11 @@ void ConsoleReporter::PrintRunData(const Run& result) { big_o.c_str(), result.cpu_accumulated_time * multiplier, big_o.c_str()); - } - else if(result.report_rms) { + } else if(result.report_rms) { ColorPrintf(COLOR_YELLOW, "%10.0f %% %10.0f %% ", result.real_accumulated_time * multiplier * 100, result.cpu_accumulated_time * multiplier * 100); - } - else if (result.iterations == 0) { + } else if (result.iterations == 0) { ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, timeLabel, @@ -144,8 +143,9 @@ void ConsoleReporter::PrintRunData(const Run& result) { timeLabel); } - if(!result.report_big_o && !result.report_rms) + if(!result.report_big_o && !result.report_rms) { ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); + } if (!rate.empty()) { ColorPrintf(COLOR_DEFAULT, " %*s", 13, rate.c_str()); diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 9bfd66b..9ac74b4 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -66,16 +66,16 @@ void CSVReporter::ReportRuns(const std::vector & reports) { } } -void CSVReporter::ReportComplexity(const std::vector & complexity_reports) { +void CSVReporter::ReportComplexity(const std::vector& complexity_reports) { if (complexity_reports.size() < 2) { // We don't report asymptotic complexity data if there was a single run. return; } - + Run big_o_data; Run rms_data; BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); - + // Output using PrintRun. PrintRunData(big_o_data); PrintRunData(rms_data); @@ -100,19 +100,19 @@ void CSVReporter::PrintRunData(const Run & run) { std::cout << "\"" << name << "\","; // Do not print iteration on bigO and RMS report - if(!run.report_big_o && !run.report_rms) - std::cout << run.iterations << ","; - else - std::cout << ","; - + if(!run.report_big_o && !run.report_rms) { + std::cout << run.iterations; + } + std::cout << ","; + std::cout << real_time << ","; std::cout << cpu_time << ","; - + // Do not print timeLabel on RMS report - if(!run.report_rms) - std::cout << timeLabel << ","; - else - std::cout << ","; + if(!run.report_rms) { + std::cout << timeLabel; + } + std::cout << ","; if (run.bytes_per_second > 0.0) { std::cout << run.bytes_per_second; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index c9d9cf1..743a223 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -120,17 +120,17 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) // We don't report asymptotic complexity data if there was a single run. return; } - + std::string indent(4, ' '); std::ostream& out = std::cout; if (!first_report_) { out << ",\n"; } - + Run big_o_data; Run rms_data; BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); - + // Output using PrintRun. out << indent << "{\n"; PrintRunData(big_o_data); diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index ea6bd46..2a73887 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -34,17 +34,21 @@ double FittingCurve(double n, benchmark::BigO complexity) { return n * log2(n); case benchmark::o1: default: - return 1; + return 1; } } -// Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. +// Internal function to find the coefficient for the high-order term in the +// running time, by minimizing the sum of squares of relative error. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. // - complexity : Fitting curve. -// For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit +// For a deeper explanation on the algorithm logic, look the README file at +// http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -LeastSq CalculateLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity) { +LeastSq CalculateLeastSq(const std::vector& n, + const std::vector& time, + const benchmark::BigO complexity) { CHECK_NE(complexity, benchmark::oAuto); double sigma_gn = 0; @@ -64,12 +68,13 @@ LeastSq CalculateLeastSq(const std::vector& n, const std::vector& t LeastSq result; result.complexity = complexity; - // Calculate complexity. + // Calculate complexity. // o1 is treated as an special case - if (complexity != benchmark::o1) + if (complexity != benchmark::o1) { result.coef = sigma_time_gn / sigma_gn_squared; - else + } else { result.coef = sigma_time / n.size(); + } // Calculate RMS double rms = 0; @@ -80,36 +85,44 @@ LeastSq CalculateLeastSq(const std::vector& n, const std::vector& t double mean = sigma_time / n.size(); - result.rms = sqrt(rms / n.size()) / mean; // Normalized RMS by the mean of the observed values + // Normalized RMS by the mean of the observed values + result.rms = sqrt(rms / n.size()) / mean; return result; } -// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - complexity : If different than oAuto, the fitting curve will stick to this one. If it is oAuto, it will be calculated -// the best fitting curve. - -LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity) { +// - complexity : If different than oAuto, the fitting curve will stick to +// this one. If it is oAuto, it will be calculated the best +// fitting curve. +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + const benchmark::BigO complexity) { CHECK_EQ(n.size(), time.size()); CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given CHECK_NE(complexity, benchmark::oNone); if(complexity == benchmark::oAuto) { - std::vector fit_curves = { benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, benchmark::oNCubed }; + std::vector fit_curves = { + benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, + benchmark::oNCubed }; - LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); // Take o1 as default best fitting curve + // Take o1 as default best fitting curve + LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); // Compute all possible fitting curves and stick to the best one for (const auto& fit : fit_curves) { LeastSq current_fit = CalculateLeastSq(n, time, fit); - if (current_fit.rms < best_fit.rms) + if (current_fit.rms < best_fit.rms) { best_fit = current_fit; + } } return best_fit; } - else - return CalculateLeastSq(n, time, complexity); -} \ No newline at end of file + + return CalculateLeastSq(n, time, complexity); +} diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index 6dcb894..0dc12b7 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -23,11 +23,13 @@ #include // This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coeficient for the high-order term as interpolated from data. +// - coef : Estimated coeficient for the high-order term as +// interpolated from data. // - rms : Normalized Root Mean Squared Error. -// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability form has been provided to MinimalLeastSq -// this will return the same value. In case BigO::oAuto has been selected, this parameter will return the -// best fitting curve detected. +// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability +// form has been provided to MinimalLeastSq this will return +// the same value. In case BigO::oAuto has been selected, this +// parameter will return the best fitting curve detected. struct LeastSq { LeastSq() : @@ -37,10 +39,13 @@ struct LeastSq { double coef; double rms; - benchmark::BigO complexity; + benchmark::BigO complexity; }; -// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. -LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity = benchmark::oAuto); +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error. +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + const benchmark::BigO complexity = benchmark::oAuto); #endif diff --git a/src/reporter.cc b/src/reporter.cc index 544df87..2830fa1 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -82,7 +82,9 @@ void BenchmarkReporter::ComputeStats( void BenchmarkReporter::ComputeBigO( const std::vector& reports, Run* big_o, Run* rms) { - CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; + CHECK(reports.size() >= 2) + << "Cannot compute asymptotic complexity for fewer than 2 reports"; + // Accumulators. std::vector n; std::vector real_time; @@ -90,21 +92,21 @@ void BenchmarkReporter::ComputeBigO( // Populate the accumulators. for (const Run& run : reports) { - n.push_back(run.complexity_n); + n.push_back(run.complexity_n); real_time.push_back(run.real_accumulated_time/run.iterations); cpu_time.push_back(run.cpu_accumulated_time/run.iterations); } - + LeastSq result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - + // result_cpu.complexity is passed as parameter to result_real because in case - // reports[0].complexity is oAuto, the noise on the measured data could make - // the best fit function of Cpu and Real differ. In order to solve this, we take - // the best fitting function for the Cpu, and apply it to Real data. + // reports[0].complexity is oAuto, the noise on the measured data could make + // the best fit function of Cpu and Real differ. In order to solve this, we + // take the best fitting function for the Cpu, and apply it to Real data. LeastSq result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); - + // Get the data from the accumulator to BenchmarkReporter::Run's. big_o->benchmark_name = benchmark_name + "_BigO"; big_o->iterations = 0; @@ -115,7 +117,8 @@ void BenchmarkReporter::ComputeBigO( double multiplier; const char* time_label; - std::tie(time_label, multiplier) = GetTimeUnitAndMultiplier(reports[0].time_unit); + std::tie(time_label, multiplier) = + GetTimeUnitAndMultiplier(reports[0].time_unit); // Only add label to mean/stddev if it is same for all runs big_o->report_label = reports[0].report_label; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index e454ee4..b8cd440 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -40,7 +40,7 @@ static void BM_Complexity_O_N(benchmark::State& state) { } BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); - + static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); std::string s2(state.range_x(), '-'); @@ -54,7 +54,7 @@ static void BM_Complexity_O_N_Squared(benchmark::State& state) { } } BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::oNSquared); - + static void BM_Complexity_O_N_Cubed(benchmark::State& state) { std::string s1(state.range_x(), '-'); std::string s2(state.range_x(), '-'); @@ -81,7 +81,7 @@ static void BM_Complexity_O_log_N(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O_log_N) +BENCHMARK(BM_Complexity_O_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oLogN); static void BM_Complexity_O_N_log_N(benchmark::State& state) { @@ -102,4 +102,4 @@ void BM_Extreme_Cases(benchmark::State& state) { BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::oNLogN); BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::oAuto); -BENCHMARK_MAIN() \ No newline at end of file +BENCHMARK_MAIN() -- cgit v1.2.3 From 1080b17bf5d0a093e5aa6fb2a474df443e05bb82 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 15:09:31 -0600 Subject: Fix build error with libc++ --- test/complexity_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/complexity_test.cc b/test/complexity_test.cc index b8cd440..6e6ae3c 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -1,6 +1,7 @@ #include "benchmark/benchmark_api.h" +#include #include #include #include -- cgit v1.2.3 From 924b8cee7af0a302ecb07b3429760c4a906460c7 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 15:21:41 -0600 Subject: Reflow some wording. NFC --- README.md | 11 ++++++----- include/benchmark/benchmark_api.h | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 15997ab..c9f08f2 100644 --- a/README.md +++ b/README.md @@ -311,11 +311,12 @@ BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); ## Exiting Benchmarks in Error -When external influences such as file I/O and network errors occur within -a benchmark the `State::SkipWithError(const char* msg)` function can be used -to skip that run of benchmark and report the error. Note that only future -iterations of the `KeepRunning()` are skipped. Users may explicitly return -to exit the benchmark immediately. +When errors caused by external influences, such as file I/O and network +communication, occur within a benchmark the +`State::SkipWithError(const char* msg)` function can be used to skip that run +of benchmark and report the error. Note that only future iterations of the +`KeepRunning()` are skipped. Users may explicitly return to exit the +benchmark immediately. The `SkipWithError(...)` function may be used at any point within the benchmark, including before and after the `KeepRunning()` loop. diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index d7718d3..77dfdd1 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -269,7 +269,7 @@ public: } // REQUIRES: timer is running and 'SkipWithError(...)' has not been called - // in the current thread. + // by the current thread. // Stop the benchmark timer. If not called, the timer will be // automatically stopped after KeepRunning() returns false for the first time. // @@ -284,7 +284,7 @@ public: void PauseTiming(); // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called - // in the current thread. + // by the current thread. // Start the benchmark timer. The timer is NOT running on entrance to the // benchmark function. It begins running after the first call to KeepRunning() // @@ -298,15 +298,15 @@ public: // within each benchmark iteration, if possible. void ResumeTiming(); - // REQUIRES: 'SkipWithError(...)' has not been called previously in the + // REQUIRES: 'SkipWithError(...)' has not been called previously by the // current thread. // Skip any future iterations of the 'KeepRunning()' loop in the current // thread and report an error with the specified 'msg'. After this call // the user may explicitly 'return' from the benchmark. // // For threaded benchmarks only the current thread stops executing. If - // multiple threads report an error only the first error message will be used. - // The current thread is no longer considered 'active' thread by + // multiple threads report an error only the first error message is used. + // The current thread is no longer considered 'active' by // 'PauseTiming()' and 'ResumingTiming()'. // // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit -- cgit v1.2.3 From 525858e68797d053ab4c859528164974978162ba Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 15:44:58 -0600 Subject: Fix error-handling in reporters --- src/csv_reporter.cc | 27 ++++++++++++++++++++------- src/json_reporter.cc | 8 +++++++- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 9ac74b4..16e13dc 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -15,6 +15,7 @@ #include "benchmark/reporter.h" #include +#include #include #include #include @@ -53,8 +54,12 @@ void CSVReporter::ReportRuns(const std::vector & reports) { return; } + auto error_count = std::count_if( + reports.begin(), reports.end(), + [](Run const& run) {return run.error_occurred;}); + std::vector reports_cp = reports; - if (reports.size() >= 2) { + if (reports.size() - error_count >= 2) { Run mean_data; Run stddev_data; BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); @@ -82,6 +87,20 @@ void CSVReporter::ReportComplexity(const std::vector& complexity_reports) { } void CSVReporter::PrintRunData(const Run & run) { + + + // Field with embedded double-quote characters must be doubled and the field + // delimited with double-quotes. + std::string name = run.benchmark_name; + ReplaceAll(&name, "\"", "\"\""); + std::cout << '"' << name << "\","; + if (run.error_occurred) { + std::cout << "error_occurred,"; + std::string msg = run.error_message; + ReplaceAll(&msg, "\"", "\"\""); + std::cout << '"' << msg << "\","; + } + double multiplier; const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); @@ -93,12 +112,6 @@ void CSVReporter::PrintRunData(const Run & run) { cpu_time = cpu_time / static_cast(run.iterations); } - // Field with embedded double-quote characters must be doubled and the field - // delimited with double-quotes. - std::string name = run.benchmark_name; - ReplaceAll(&name, "\"", "\"\""); - std::cout << "\"" << name << "\","; - // Do not print iteration on bigO and RMS report if(!run.report_big_o && !run.report_rms) { std::cout << run.iterations; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 4979643..8248753 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -15,6 +15,7 @@ #include "benchmark/reporter.h" #include +#include #include #include #include @@ -96,8 +97,13 @@ void JSONReporter::ReportRuns(std::vector const& reports) { out << ",\n"; } first_report_ = false; + + auto error_count = std::count_if( + reports.begin(), reports.end(), + [](Run const& run) {return run.error_occurred;}); + std::vector reports_cp = reports; - if (reports.size() >= 2) { + if (reports.size() - error_count >= 2) { Run mean_data; Run stddev_data; BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); -- cgit v1.2.3 From 6a45324e1d1c1f2c7e56a2df970e38a62122fc4f Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 16:12:30 -0600 Subject: Address review comments. --- src/benchmark.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 7532c9c..1b966d4 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -206,11 +206,10 @@ class TimerManager { void RemoveErroredThread() EXCLUDES(lock_) { MutexLock ml(lock_); int last_thread = --running_threads_ == 0; - if (last_thread && running_) { - InternalStop(); - } else if (!last_thread){ + if (last_thread && running_) + InternalStop(); + else if (!last_thread) phase_condition_.notify_all(); - } } // REQUIRES: timer is not running @@ -279,13 +278,10 @@ class TimerManager { return this->phase_number_ > phase_number_cp || entered_ == running_threads_; // A thread has aborted in error }; - while (true) { - phase_condition_.wait(ml.native_handle(), cb); - if (phase_number_ > phase_number_cp) + phase_condition_.wait(ml.native_handle(), cb); + if (phase_number_ > phase_number_cp) return false; - else if (running_threads_ == entered_) - break; - } + // else (running_threads_ == entered_) and we are the last thread. } // Last thread has reached the barrier phase_number_++; -- cgit v1.2.3 From ee8e37c67db9a717059f9f7bdfb83549208d65e9 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 16:19:04 -0600 Subject: Fix bad merge which caused the benchmark name to be printed twice --- src/console_reporter.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 2b1f281..3944666 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -100,11 +100,14 @@ void ConsoleReporter::ReportComplexity(const std::vector & complexity_repor } void ConsoleReporter::PrintRunData(const Run& result) { - ColorPrintf(COLOR_GREEN, "%-*s ", - name_field_width_, result.benchmark_name.c_str()); + auto name_color = (result.report_big_o || result.report_rms) + ? COLOR_BLUE : COLOR_GREEN; + ColorPrintf(name_color, "%-*s ", name_field_width_, + result.benchmark_name.c_str()); if (result.error_occurred) { - ColorPrintf(COLOR_RED, "ERROR OCCURRED: \'%s\'", result.error_message.c_str()); + ColorPrintf(COLOR_RED, "ERROR OCCURRED: \'%s\'", + result.error_message.c_str()); ColorPrintf(COLOR_DEFAULT, "\n"); return; } @@ -125,10 +128,6 @@ void ConsoleReporter::PrintRunData(const Run& result) { const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(result.time_unit); - ColorPrintf((result.report_big_o ||result.report_rms) ? COLOR_BLUE : - COLOR_GREEN, "%-*s ", - name_field_width_, result.benchmark_name.c_str()); - if(result.report_big_o) { std::string big_o = result.report_big_o ? GetBigO(result.complexity) : ""; ColorPrintf(COLOR_YELLOW, "%10.4f %s %10.4f %s ", -- cgit v1.2.3 From 6f84ffcd8b53b035fe4430070dbca19641892def Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 18:25:44 -0600 Subject: fix another bad merge --- src/json_reporter.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 8248753..cab527c 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -176,9 +176,6 @@ void JSONReporter::PrintRunData(Run const& run) { << FormatKV("error_message", run.error_message) << ",\n"; } - out << indent - << FormatKV("iterations", run.iterations) - << ",\n"; if(!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) -- cgit v1.2.3 From 1003a70e5fe80967fe465da54c2f924944ad9fa6 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 19:45:18 -0600 Subject: Fix csv_reporter when reporting errors --- src/csv_reporter.cc | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 16e13dc..ac4bd3d 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -28,6 +28,21 @@ namespace benchmark { +namespace { +std::vector elements = { + "name", + "iterations", + "real_time", + "cpu_time", + "time_unit", + "bytes_per_second", + "items_per_second", + "label", + "error_occurred", + "error_message" +}; +} + bool CSVReporter::ReportContext(const Context& context) { std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; @@ -44,8 +59,12 @@ bool CSVReporter::ReportContext(const Context& context) { std::cerr << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; #endif - std::cout << "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," - "items_per_second,label\n"; + for (auto B = elements.begin(); B != elements.end(); ) { + std::cout << *B++; + if (B != elements.end()) + std::cout << ","; + } + std::cout << "\n"; return true; } @@ -95,10 +114,12 @@ void CSVReporter::PrintRunData(const Run & run) { ReplaceAll(&name, "\"", "\"\""); std::cout << '"' << name << "\","; if (run.error_occurred) { - std::cout << "error_occurred,"; + std::cout << std::string(elements.size() - 3, ','); + std::cout << "true,"; std::string msg = run.error_message; ReplaceAll(&msg, "\"", "\"\""); - std::cout << '"' << msg << "\","; + std::cout << '"' << msg << "\"\n"; + return; } double multiplier; @@ -142,6 +163,7 @@ void CSVReporter::PrintRunData(const Run & run) { ReplaceAll(&label, "\"", "\"\""); std::cout << "\"" << label << "\""; } + std::cout << ",,"; // for error_occurred and error_message std::cout << '\n'; } -- cgit v1.2.3 From 84bc4d703b6f27a0bdcb48443c9bca3f60e1818c Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 24 May 2016 21:52:23 -0600 Subject: Add a per benchmark 'Repetitions' option. --- README.md | 11 +++++++++++ include/benchmark/benchmark_api.h | 6 ++++++ src/benchmark.cc | 29 ++++++++++++++++++++++++++--- test/options_test.cc | 1 + 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 498c4ca..b03d222 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,17 @@ the minimum time, or the wallclock time is 5x minimum time. The minimum time is set as a flag `--benchmark_min_time` or per benchmark by calling `MinTime` on the registered benchmark object. +## Reporting the mean and standard devation by repeated benchmarks +By default each benchmark is run once and that single result is reported. +However benchmarks are often noisy and a single result may not be representative +of the overall behavior. For this reason it's possible to repeatedly rerun the +benchmark. + +The number of runs of each benchmark is specified globally by the +`--benchmark_repetitions` flag or on a per benchmark basis by calling +`Repetitions` on the registered benchmark object. When a benchmark is run +more than once the mean and standard deviation of the runs will be reported. + ## Fixtures Fixture tests are created by first defining a type that derives from ::benchmark::Fixture and then diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 674b0b5..2fe8753 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -477,8 +477,14 @@ public: // Set the minimum amount of time to use when running this benchmark. This // option overrides the `benchmark_min_time` flag. + // REQUIRES: `t > 0` Benchmark* MinTime(double t); + // Specify the amount of times to repeat this benchmark. This option overrides + // the `benchmark_repetitions` flag. + // REQUIRES: `n > 0` + Benchmark* Repetitions(int n); + // If a particular benchmark is I/O bound, runs multiple threads internally or // if for some reason CPU timings are not representative, call this method. If // called, the elapsed time will be used to control how many iterations are diff --git a/src/benchmark.cc b/src/benchmark.cc index bd9858f..41d529d 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -293,6 +293,7 @@ struct Benchmark::Instance { bool use_manual_time; BigO complexity; bool last_benchmark_instance; + int repetitions; double min_time; int threads; // Number of concurrent threads to use bool multithreaded; // Is benchmark multi-threaded? @@ -332,6 +333,7 @@ public: void RangePair(int lo1, int hi1, int lo2, int hi2); void RangeMultiplier(int multiplier); void MinTime(double n); + void Repetitions(int n); void UseRealTime(); void UseManualTime(); void Complexity(BigO complexity); @@ -351,6 +353,7 @@ private: TimeUnit time_unit_; int range_multiplier_; double min_time_; + int repetitions_; bool use_real_time_; bool use_manual_time_; BigO complexity_; @@ -414,6 +417,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.time_unit = family->time_unit_; instance.range_multiplier = family->range_multiplier_; instance.min_time = family->min_time_; + instance.repetitions = family->repetitions_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; instance.complexity = family->complexity_; @@ -430,6 +434,9 @@ bool BenchmarkFamilies::FindBenchmarks( if (!IsZero(family->min_time_)) { instance.name += StringPrintF("/min_time:%0.3f", family->min_time_); } + if (family->repetitions_ != 0) { + instance.name += StringPrintF("/repeats:%d", family->repetitions_); + } if (family->use_manual_time_) { instance.name += "/manual_time"; } else if (family->use_real_time_) { @@ -453,7 +460,7 @@ bool BenchmarkFamilies::FindBenchmarks( BenchmarkImp::BenchmarkImp(const char* name) : name_(name), arg_count_(-1), time_unit_(kNanosecond), - range_multiplier_(kRangeMultiplier), min_time_(0.0), + range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0), use_real_time_(false), use_manual_time_(false), complexity_(oNone) { } @@ -521,6 +528,12 @@ void BenchmarkImp::MinTime(double t) { min_time_ = t; } + +void BenchmarkImp::Repetitions(int n) { + CHECK(n > 0); + repetitions_ = n; +} + void BenchmarkImp::UseRealTime() { CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; use_real_time_ = true; @@ -633,6 +646,12 @@ Benchmark* Benchmark::RangeMultiplier(int multiplier) { return this; } + +Benchmark* Benchmark::Repetitions(int t) { + imp_->Repetitions(t); + return this; +} + Benchmark* Benchmark::MinTime(double t) { imp_->MinTime(t); return this; @@ -712,7 +731,9 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, if (b.multithreaded) pool.resize(b.threads); - for (int i = 0; i < FLAGS_benchmark_repetitions; i++) { + const int repeats = b.repetitions != 0 ? b.repetitions + : FLAGS_benchmark_repetitions; + for (int i = 0; i < repeats; i++) { std::string mem; for (;;) { // Try benchmark @@ -893,12 +914,14 @@ void RunMatchingBenchmarks(const std::vector& benchmarks, CHECK(reporter != nullptr); // Determine the width of the name field using a minimum width of 10. + bool has_repetitions = FLAGS_benchmark_repetitions > 1; size_t name_field_width = 10; for (const Benchmark::Instance& benchmark : benchmarks) { name_field_width = std::max(name_field_width, benchmark.name.size()); + has_repetitions |= benchmark.repetitions > 1; } - if (FLAGS_benchmark_repetitions > 1) + if (has_repetitions) name_field_width += std::strlen("_stddev"); // Print header here diff --git a/test/options_test.cc b/test/options_test.cc index ed5fa68..78cedae 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -31,6 +31,7 @@ BENCHMARK(BM_basic)->MinTime(0.7); BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); BENCHMARK(BM_basic)->ThreadPerCpu(); +BENCHMARK(BM_basic)->Repetitions(3); void CustomArgs(benchmark::internal::Benchmark* b) { for (int i = 0; i < 10; ++i) { -- cgit v1.2.3 From e246699f25f50e80fad71a1352313f5f3d3e21d3 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 21:18:56 +0200 Subject: added auto as default value for complexity --- include/benchmark/benchmark_api.h | 2 +- test/complexity_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 2fe8753..9c32bac 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -501,7 +501,7 @@ public: // Set the asymptotic computational complexity for the benchmark. If called // the asymptotic computational complexity will be shown on the output. - Benchmark* Complexity(BigO complexity); + Benchmark* Complexity(BigO complexity = benchmark::oAuto); // Support for running multiple copies of the same benchmark concurrently // in multiple threads. This may be useful when measuring the scaling diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 6e6ae3c..35a6fa0 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -40,7 +40,7 @@ static void BM_Complexity_O_N(benchmark::State& state) { state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); static void BM_Complexity_O_N_Squared(benchmark::State& state) { std::string s1(state.range_x(), '-'); -- cgit v1.2.3 From 087f0d3f1bc6610ceaa346f8e573dd23236cea08 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 22:26:57 +0200 Subject: upgraded leastsq --- src/minimal_leastsq.cc | 84 ++++++++++++++++++++++++++++++-------------------- src/minimal_leastsq.h | 19 +++++++++++- 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc index 2a73887..d63e6f6 100644 --- a/src/minimal_leastsq.cc +++ b/src/minimal_leastsq.cc @@ -20,37 +20,54 @@ #include // Internal function to calculate the different scalability forms -double FittingCurve(double n, benchmark::BigO complexity) { +std::function FittingCurve(benchmark::BigO complexity) { + switch (complexity) { + case benchmark::oN: + return [](int n) {return n; }; + case benchmark::oNSquared: + return [](int n) {return n*n; }; + case benchmark::oNCubed: + return [](int n) {return n*n*n; }; + case benchmark::oLogN: + return [](int n) {return log2(n); }; + case benchmark::oNLogN: + return [](int n) {return n * log2(n); }; + case benchmark::o1: + default: + return [](int) {return 1; }; + } +} + +// Internal function to to return an string for the calculated complexity +std::string GetBigOString(benchmark::BigO complexity) { switch (complexity) { case benchmark::oN: - return n; + return "* N"; case benchmark::oNSquared: - return pow(n, 2); + return "* N**2"; case benchmark::oNCubed: - return pow(n, 3); + return "* N**3"; case benchmark::oLogN: - return log2(n); + return "* lgN"; case benchmark::oNLogN: - return n * log2(n); + return "* NlgN"; case benchmark::o1: + return "* 1"; default: - return 1; + return ""; } } -// Internal function to find the coefficient for the high-order term in the -// running time, by minimizing the sum of squares of relative error. -// - n : Vector containing the size of the benchmark tests. -// - time : Vector containing the times for the benchmark tests. -// - complexity : Fitting curve. +// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error, for the fitting curve given on the lambda expresion. +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). // For a deeper explanation on the algorithm logic, look the README file at // http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -LeastSq CalculateLeastSq(const std::vector& n, - const std::vector& time, - const benchmark::BigO complexity) { - CHECK_NE(complexity, benchmark::oAuto); - +LeastSq CalculateLeastSq(const std::vector& n, + const std::vector& time, + std::function fitting_curve) { double sigma_gn = 0; double sigma_gn_squared = 0; double sigma_time = 0; @@ -58,7 +75,7 @@ LeastSq CalculateLeastSq(const std::vector& n, // Calculate least square fitting parameter for (size_t i = 0; i < n.size(); ++i) { - double gn_i = FittingCurve(n[i], complexity); + double gn_i = fitting_curve(n[i]); sigma_gn += gn_i; sigma_gn_squared += gn_i * gn_i; sigma_time += time[i]; @@ -66,26 +83,19 @@ LeastSq CalculateLeastSq(const std::vector& n, } LeastSq result; - result.complexity = complexity; // Calculate complexity. - // o1 is treated as an special case - if (complexity != benchmark::o1) { - result.coef = sigma_time_gn / sigma_gn_squared; - } else { - result.coef = sigma_time / n.size(); - } + result.coef = sigma_time_gn / sigma_gn_squared; // Calculate RMS double rms = 0; for (size_t i = 0; i < n.size(); ++i) { - double fit = result.coef * FittingCurve(n[i], complexity); + double fit = result.coef * fitting_curve(n[i]); rms += pow((time[i] - fit), 2); } - double mean = sigma_time / n.size(); - // Normalized RMS by the mean of the observed values + double mean = sigma_time / n.size(); result.rms = sqrt(rms / n.size()) / mean; return result; @@ -105,24 +115,32 @@ LeastSq MinimalLeastSq(const std::vector& n, CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given CHECK_NE(complexity, benchmark::oNone); + LeastSq best_fit; + if(complexity == benchmark::oAuto) { std::vector fit_curves = { benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, benchmark::oNCubed }; // Take o1 as default best fitting curve - LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); + best_fit = CalculateLeastSq(n, time, FittingCurve(benchmark::o1)); + best_fit.complexity = benchmark::o1; + best_fit.caption = GetBigOString(benchmark::o1); // Compute all possible fitting curves and stick to the best one for (const auto& fit : fit_curves) { - LeastSq current_fit = CalculateLeastSq(n, time, fit); + LeastSq current_fit = CalculateLeastSq(n, time, FittingCurve(fit)); if (current_fit.rms < best_fit.rms) { best_fit = current_fit; + best_fit.complexity = fit; + best_fit.caption = GetBigOString(fit); } } - - return best_fit; + } else { + best_fit = CalculateLeastSq(n, time, FittingCurve(complexity)); + best_fit.complexity = complexity; + best_fit.caption = GetBigOString(complexity); } - return CalculateLeastSq(n, time, complexity); + return best_fit; } diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h index 0dc12b7..ee49ec9 100644 --- a/src/minimal_leastsq.h +++ b/src/minimal_leastsq.h @@ -21,6 +21,7 @@ #include "benchmark/benchmark_api.h" #include +#include // This data structure will contain the result returned by MinimalLeastSq // - coef : Estimated coeficient for the high-order term as @@ -35,11 +36,13 @@ struct LeastSq { LeastSq() : coef(0), rms(0), - complexity(benchmark::oNone) {} + complexity(benchmark::oNone), + caption("") {} double coef; double rms; benchmark::BigO complexity; + std::string caption; }; // Find the coefficient for the high-order term in the running time, by @@ -48,4 +51,18 @@ LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const benchmark::BigO complexity = benchmark::oAuto); +// This interface is currently not used from the oustide, but it has been provided +// for future upgrades. If in the future it is not needed to support Cxx03, then +// all the calculations could be upgraded to use lambdas because they are more +// powerful and provide a cleaner inferface than enumerators, but complete +// implementation with lambdas will not work for Cxx03 (e.g. lack of std::function). +// In case lambdas are implemented, the interface would be like : +// -> Complexity([](int n) {return n;};) +// and any arbitrary and valid equation would be allowed, but the option to calculate +// the best fit to the most common scalability curves will still be kept. +LeastSq CalculateLeastSq(const std::vector& n, + const std::vector& time, + std::function fitting_curve); + + #endif -- cgit v1.2.3 From 2f61f8aee0bc8b09429fce8b7d2718f805ed18ac Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 22:57:52 +0200 Subject: refactor leastsq into complexity --- include/benchmark/complexity.h | 65 ++++++++++++----- src/CMakeLists.txt | 2 +- src/complexity.cc | 156 +++++++++++++++++++++++++++++++++++++++++ src/console_reporter.cc | 2 +- src/minimal_leastsq.cc | 146 -------------------------------------- src/minimal_leastsq.h | 68 ------------------ src/reporter.cc | 1 - 7 files changed, 205 insertions(+), 235 deletions(-) create mode 100644 src/complexity.cc delete mode 100644 src/minimal_leastsq.cc delete mode 100644 src/minimal_leastsq.h diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h index 93b26de..1c9f591 100644 --- a/include/benchmark/complexity.h +++ b/include/benchmark/complexity.h @@ -1,7 +1,26 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Adapted to be used with google benchmark + #ifndef COMPLEXITY_H_ #define COMPLEXITY_H_ #include +#include +#include namespace benchmark { @@ -19,24 +38,34 @@ enum BigO { oAuto }; -inline std::string GetBigO(BigO complexity) { - switch (complexity) { - case oN: - return "* N"; - case oNSquared: - return "* N**2"; - case oNCubed: - return "* N**3"; - case oLogN: - return "* lgN"; - case oNLogN: - return "* NlgN"; - case o1: - return "* 1"; - default: - return ""; - } -} +// This data structure will contain the result returned by MinimalLeastSq +// - coef : Estimated coeficient for the high-order term as +// interpolated from data. +// - rms : Normalized Root Mean Squared Error. +// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability +// form has been provided to MinimalLeastSq this will return +// the same value. In case BigO::oAuto has been selected, this +// parameter will return the best fitting curve detected. + +struct LeastSq { + LeastSq() : + coef(0), + rms(0), + complexity(benchmark::oNone) {} + + double coef; + double rms; + benchmark::BigO complexity; +}; + +// Function to to return an string for the calculated complexity +std::string GetBigOString(benchmark::BigO complexity); + +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error. +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + const benchmark::BigO complexity = benchmark::oAuto); } // end namespace benchmark #endif // COMPLEXITY_H_ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a681b35..6dab64b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories(${PROJECT_SOURCE_DIR}/src) set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc" "console_reporter.cc" "csv_reporter.cc" "json_reporter.cc" "log.cc" "reporter.cc" "sleep.cc" "string_util.cc" - "sysinfo.cc" "walltime.cc" "minimal_leastsq.cc") + "sysinfo.cc" "walltime.cc" "complexity.cc") # Determine the correct regular expression engine to use if(HAVE_STD_REGEX) set(RE_FILES "re_std.cc") diff --git a/src/complexity.cc b/src/complexity.cc new file mode 100644 index 0000000..4d21af8 --- /dev/null +++ b/src/complexity.cc @@ -0,0 +1,156 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Adapted to be used with google benchmark + +#include "benchmark/complexity.h" +#include "check.h" +#include + +namespace benchmark { + +// Internal function to calculate the different scalability forms +std::function FittingCurve(BigO complexity) { + switch (complexity) { + case oN: + return [](int n) {return n; }; + case oNSquared: + return [](int n) {return n*n; }; + case oNCubed: + return [](int n) {return n*n*n; }; + case oLogN: + return [](int n) {return log2(n); }; + case oNLogN: + return [](int n) {return n * log2(n); }; + case o1: + default: + return [](int) {return 1; }; + } +} + +// Function to to return an string for the calculated complexity +std::string GetBigOString(BigO complexity) { + switch (complexity) { + case oN: + return "* N"; + case oNSquared: + return "* N**2"; + case oNCubed: + return "* N**3"; + case oLogN: + return "* lgN"; + case oNLogN: + return "* NlgN"; + case o1: + return "* 1"; + default: + return ""; + } +} + +// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error, for the fitting curve given on the lambda expresion. +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). +// For a deeper explanation on the algorithm logic, look the README file at +// http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit + +// This interface is currently not used from the oustide, but it has been provided +// for future upgrades. If in the future it is not needed to support Cxx03, then +// all the calculations could be upgraded to use lambdas because they are more +// powerful and provide a cleaner inferface than enumerators, but complete +// implementation with lambdas will not work for Cxx03 (e.g. lack of std::function). +// In case lambdas are implemented, the interface would be like : +// -> Complexity([](int n) {return n;};) +// and any arbitrary and valid equation would be allowed, but the option to calculate +// the best fit to the most common scalability curves will still be kept. + +LeastSq CalculateLeastSq(const std::vector& n, + const std::vector& time, + std::function fitting_curve) { + double sigma_gn = 0; + double sigma_gn_squared = 0; + double sigma_time = 0; + double sigma_time_gn = 0; + + // Calculate least square fitting parameter + for (size_t i = 0; i < n.size(); ++i) { + double gn_i = fitting_curve(n[i]); + sigma_gn += gn_i; + sigma_gn_squared += gn_i * gn_i; + sigma_time += time[i]; + sigma_time_gn += time[i] * gn_i; + } + + LeastSq result; + + // Calculate complexity. + result.coef = sigma_time_gn / sigma_gn_squared; + + // Calculate RMS + double rms = 0; + for (size_t i = 0; i < n.size(); ++i) { + double fit = result.coef * fitting_curve(n[i]); + rms += pow((time[i] - fit), 2); + } + + // Normalized RMS by the mean of the observed values + double mean = sigma_time / n.size(); + result.rms = sqrt(rms / n.size()) / mean; + + return result; +} + +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error. +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - complexity : If different than oAuto, the fitting curve will stick to +// this one. If it is oAuto, it will be calculated the best +// fitting curve. +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + const BigO complexity) { + CHECK_EQ(n.size(), time.size()); + CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given + CHECK_NE(complexity, oNone); + + LeastSq best_fit; + + if(complexity == oAuto) { + std::vector fit_curves = { + oLogN, oN, oNLogN, oNSquared, oNCubed }; + + // Take o1 as default best fitting curve + best_fit = CalculateLeastSq(n, time, FittingCurve(o1)); + best_fit.complexity = o1; + + // Compute all possible fitting curves and stick to the best one + for (const auto& fit : fit_curves) { + LeastSq current_fit = CalculateLeastSq(n, time, FittingCurve(fit)); + if (current_fit.rms < best_fit.rms) { + best_fit = current_fit; + best_fit.complexity = fit; + } + } + } else { + best_fit = CalculateLeastSq(n, time, FittingCurve(complexity)); + best_fit.complexity = complexity; + } + + return best_fit; +} + +} // end namespace benchmark \ No newline at end of file diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 41c00b9..da07519 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -117,7 +117,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { name_field_width_, result.benchmark_name.c_str()); if(result.report_big_o) { - std::string big_o = result.report_big_o ? GetBigO(result.complexity) : ""; + std::string big_o = result.report_big_o ? GetBigOString(result.complexity) : ""; ColorPrintf(COLOR_YELLOW, "%10.4f %s %10.4f %s ", result.real_accumulated_time * multiplier, big_o.c_str(), diff --git a/src/minimal_leastsq.cc b/src/minimal_leastsq.cc deleted file mode 100644 index d63e6f6..0000000 --- a/src/minimal_leastsq.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Source project : https://github.com/ismaelJimenez/cpp.leastsq -// Adapted to be used with google benchmark - -#include "minimal_leastsq.h" -#include "check.h" -#include - -// Internal function to calculate the different scalability forms -std::function FittingCurve(benchmark::BigO complexity) { - switch (complexity) { - case benchmark::oN: - return [](int n) {return n; }; - case benchmark::oNSquared: - return [](int n) {return n*n; }; - case benchmark::oNCubed: - return [](int n) {return n*n*n; }; - case benchmark::oLogN: - return [](int n) {return log2(n); }; - case benchmark::oNLogN: - return [](int n) {return n * log2(n); }; - case benchmark::o1: - default: - return [](int) {return 1; }; - } -} - -// Internal function to to return an string for the calculated complexity -std::string GetBigOString(benchmark::BigO complexity) { - switch (complexity) { - case benchmark::oN: - return "* N"; - case benchmark::oNSquared: - return "* N**2"; - case benchmark::oNCubed: - return "* N**3"; - case benchmark::oLogN: - return "* lgN"; - case benchmark::oNLogN: - return "* NlgN"; - case benchmark::o1: - return "* 1"; - default: - return ""; - } -} - -// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error, for the fitting curve given on the lambda expresion. -// - n : Vector containing the size of the benchmark tests. -// - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). -// For a deeper explanation on the algorithm logic, look the README file at -// http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit - -LeastSq CalculateLeastSq(const std::vector& n, - const std::vector& time, - std::function fitting_curve) { - double sigma_gn = 0; - double sigma_gn_squared = 0; - double sigma_time = 0; - double sigma_time_gn = 0; - - // Calculate least square fitting parameter - for (size_t i = 0; i < n.size(); ++i) { - double gn_i = fitting_curve(n[i]); - sigma_gn += gn_i; - sigma_gn_squared += gn_i * gn_i; - sigma_time += time[i]; - sigma_time_gn += time[i] * gn_i; - } - - LeastSq result; - - // Calculate complexity. - result.coef = sigma_time_gn / sigma_gn_squared; - - // Calculate RMS - double rms = 0; - for (size_t i = 0; i < n.size(); ++i) { - double fit = result.coef * fitting_curve(n[i]); - rms += pow((time[i] - fit), 2); - } - - // Normalized RMS by the mean of the observed values - double mean = sigma_time / n.size(); - result.rms = sqrt(rms / n.size()) / mean; - - return result; -} - -// Find the coefficient for the high-order term in the running time, by -// minimizing the sum of squares of relative error. -// - n : Vector containing the size of the benchmark tests. -// - time : Vector containing the times for the benchmark tests. -// - complexity : If different than oAuto, the fitting curve will stick to -// this one. If it is oAuto, it will be calculated the best -// fitting curve. -LeastSq MinimalLeastSq(const std::vector& n, - const std::vector& time, - const benchmark::BigO complexity) { - CHECK_EQ(n.size(), time.size()); - CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given - CHECK_NE(complexity, benchmark::oNone); - - LeastSq best_fit; - - if(complexity == benchmark::oAuto) { - std::vector fit_curves = { - benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, - benchmark::oNCubed }; - - // Take o1 as default best fitting curve - best_fit = CalculateLeastSq(n, time, FittingCurve(benchmark::o1)); - best_fit.complexity = benchmark::o1; - best_fit.caption = GetBigOString(benchmark::o1); - - // Compute all possible fitting curves and stick to the best one - for (const auto& fit : fit_curves) { - LeastSq current_fit = CalculateLeastSq(n, time, FittingCurve(fit)); - if (current_fit.rms < best_fit.rms) { - best_fit = current_fit; - best_fit.complexity = fit; - best_fit.caption = GetBigOString(fit); - } - } - } else { - best_fit = CalculateLeastSq(n, time, FittingCurve(complexity)); - best_fit.complexity = complexity; - best_fit.caption = GetBigOString(complexity); - } - - return best_fit; -} diff --git a/src/minimal_leastsq.h b/src/minimal_leastsq.h deleted file mode 100644 index ee49ec9..0000000 --- a/src/minimal_leastsq.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Source project : https://github.com/ismaelJimenez/cpp.leastsq -// Adapted to be used with google benchmark - -#if !defined(MINIMAL_LEASTSQ_H_) -#define MINIMAL_LEASTSQ_H_ - -#include "benchmark/benchmark_api.h" - -#include -#include - -// This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coeficient for the high-order term as -// interpolated from data. -// - rms : Normalized Root Mean Squared Error. -// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability -// form has been provided to MinimalLeastSq this will return -// the same value. In case BigO::oAuto has been selected, this -// parameter will return the best fitting curve detected. - -struct LeastSq { - LeastSq() : - coef(0), - rms(0), - complexity(benchmark::oNone), - caption("") {} - - double coef; - double rms; - benchmark::BigO complexity; - std::string caption; -}; - -// Find the coefficient for the high-order term in the running time, by -// minimizing the sum of squares of relative error. -LeastSq MinimalLeastSq(const std::vector& n, - const std::vector& time, - const benchmark::BigO complexity = benchmark::oAuto); - -// This interface is currently not used from the oustide, but it has been provided -// for future upgrades. If in the future it is not needed to support Cxx03, then -// all the calculations could be upgraded to use lambdas because they are more -// powerful and provide a cleaner inferface than enumerators, but complete -// implementation with lambdas will not work for Cxx03 (e.g. lack of std::function). -// In case lambdas are implemented, the interface would be like : -// -> Complexity([](int n) {return n;};) -// and any arbitrary and valid equation would be allowed, but the option to calculate -// the best fit to the most common scalability curves will still be kept. -LeastSq CalculateLeastSq(const std::vector& n, - const std::vector& time, - std::function fitting_curve); - - -#endif diff --git a/src/reporter.cc b/src/reporter.cc index 2830fa1..4fd0ba5 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -13,7 +13,6 @@ // limitations under the License. #include "benchmark/reporter.h" -#include "minimal_leastsq.h" #include #include -- cgit v1.2.3 From 90a85080636d0626ed975531d08bc1339a405fa9 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 23:06:27 +0200 Subject: Update Readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b03d222..66706eb 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ calculated automatically. ```c++ BENCHMARK(BM_StringCompare) - ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oAuto); + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(); ``` ### Templated benchmarks -- cgit v1.2.3 From 1ee11056c1f1117142af36dd3ac4df2c2e6ce1bb Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 23:13:19 +0200 Subject: move include from .h into .cc --- include/benchmark/complexity.h | 1 - src/complexity.cc | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h index 1c9f591..72bd9e4 100644 --- a/include/benchmark/complexity.h +++ b/include/benchmark/complexity.h @@ -20,7 +20,6 @@ #include #include -#include namespace benchmark { diff --git a/src/complexity.cc b/src/complexity.cc index 4d21af8..67d8a05 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -18,6 +18,7 @@ #include "benchmark/complexity.h" #include "check.h" #include +#include namespace benchmark { @@ -153,4 +154,4 @@ LeastSq MinimalLeastSq(const std::vector& n, return best_fit; } -} // end namespace benchmark \ No newline at end of file +} // end namespace benchmark -- cgit v1.2.3 From 290ac9ee0ed445811897c790715e47d490dacd9f Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 23:19:32 +0200 Subject: updated complexity_test.cc to new interface for auto --- test/complexity_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 35a6fa0..225a181 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -93,7 +93,7 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); // Test benchmark with no range and check no complexity is calculated. void BM_Extreme_Cases(benchmark::State& state) { @@ -101,6 +101,6 @@ void BM_Extreme_Cases(benchmark::State& state) { } } BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::oNLogN); -BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::oAuto); +BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(); BENCHMARK_MAIN() -- cgit v1.2.3 From 340fe557e2995addcb2af24dfdc7d86801487330 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 23:22:53 +0200 Subject: indent --- src/complexity.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index 67d8a05..ad59ce8 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -25,19 +25,19 @@ namespace benchmark { // Internal function to calculate the different scalability forms std::function FittingCurve(BigO complexity) { switch (complexity) { - case oN: - return [](int n) {return n; }; - case oNSquared: - return [](int n) {return n*n; }; - case oNCubed: - return [](int n) {return n*n*n; }; - case oLogN: - return [](int n) {return log2(n); }; - case oNLogN: - return [](int n) {return n * log2(n); }; - case o1: - default: - return [](int) {return 1; }; + case oN: + return [](int n) {return n; }; + case oNSquared: + return [](int n) {return n*n; }; + case oNCubed: + return [](int n) {return n*n*n; }; + case oLogN: + return [](int n) {return log2(n); }; + case oNLogN: + return [](int n) {return n * log2(n); }; + case o1: + default: + return [](int) {return 1; }; } } -- cgit v1.2.3 From 171588561112744263caa5847847e76e9bbde562 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 25 May 2016 23:33:25 +0200 Subject: fixed typos --- include/benchmark/complexity.h | 2 +- src/complexity.cc | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h index 72bd9e4..f2eaefa 100644 --- a/include/benchmark/complexity.h +++ b/include/benchmark/complexity.h @@ -57,7 +57,7 @@ struct LeastSq { benchmark::BigO complexity; }; -// Function to to return an string for the calculated complexity +// Function to return an string for the calculated complexity std::string GetBigOString(benchmark::BigO complexity); // Find the coefficient for the high-order term in the running time, by diff --git a/src/complexity.cc b/src/complexity.cc index ad59ce8..8fa75b3 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -41,7 +41,7 @@ std::function FittingCurve(BigO complexity) { } } -// Function to to return an string for the calculated complexity +// Function to return an string for the calculated complexity std::string GetBigOString(BigO complexity) { switch (complexity) { case oN: @@ -61,22 +61,27 @@ std::string GetBigOString(BigO complexity) { } } -// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error, for the fitting curve given on the lambda expresion. +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error, for the fitting curve +// given by the lambda expresion. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. // - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). + // For a deeper explanation on the algorithm logic, look the README file at // http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -// This interface is currently not used from the oustide, but it has been provided -// for future upgrades. If in the future it is not needed to support Cxx03, then -// all the calculations could be upgraded to use lambdas because they are more -// powerful and provide a cleaner inferface than enumerators, but complete -// implementation with lambdas will not work for Cxx03 (e.g. lack of std::function). +// This interface is currently not used from the oustide, but it has been +// provided for future upgrades. If in the future it is not needed to support +// Cxx03, then all the calculations could be upgraded to use lambdas because +// they are more powerful and provide a cleaner inferface than enumerators, +// but complete implementation with lambdas will not work for Cxx03 +// (e.g. lack of std::function). // In case lambdas are implemented, the interface would be like : // -> Complexity([](int n) {return n;};) -// and any arbitrary and valid equation would be allowed, but the option to calculate -// the best fit to the most common scalability curves will still be kept. +// and any arbitrary and valid equation would be allowed, but the option to +// calculate the best fit to the most common scalability curves will still +// be kept. LeastSq CalculateLeastSq(const std::vector& n, const std::vector& time, -- cgit v1.2.3 From 74e82e822f55871a969b1642019d57639d9a4eb4 Mon Sep 17 00:00:00 2001 From: Albert Pretorius Date: Wed, 25 May 2016 07:31:20 +0100 Subject: Force DoNotOptimize operand to memory for both gcc and clang --- AUTHORS | 1 + CONTRIBUTORS | 1 + include/benchmark/benchmark_api.h | 10 ++++------ test/CMakeLists.txt | 3 +++ test/donotoptimize_test.cc | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 test/donotoptimize_test.cc diff --git a/AUTHORS b/AUTHORS index 7ddffd8..0f93e01 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,6 +8,7 @@ # # Please keep the list sorted. +Albert Pretorius Arne Beer Christopher Seymour David Coeurjolly diff --git a/CONTRIBUTORS b/CONTRIBUTORS index af40292..4bff126 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -22,6 +22,7 @@ # # Please keep the list sorted. +Albert Pretorius Arne Beer Billy Robert O'Neal III Chris Kennelly diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 9bf38a8..4167553 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -210,20 +210,18 @@ Benchmark* RegisterBenchmarkInternal(Benchmark*); // expression from being optimized away by the compiler. This function is // intented to add little to no overhead. // See: http://stackoverflow.com/questions/28287064 -#if defined(__clang__) && defined(__GNUC__) +#if defined(__GNUC__) // TODO(ericwf): Clang has a bug where it tries to always use a register // even if value must be stored in memory. This causes codegen to fail. // To work around this we remove the "r" modifier so the operand is always // loaded into memory. +// GCC also has a bug where it complains about inconsistent operand constraints +// when "+rm" is used for a type larger than can fit in a register or two. +// For now force the operand to memory for both GCC and Clang. template inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { asm volatile("" : "+m" (const_cast(value))); } -#elif defined(__GNUC__) -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : "+rm" (const_cast(value))); -} #else template inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7d75c11..247c630 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,9 @@ add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01) compile_benchmark_test(skip_with_error_test) add_test(skip_with_error_test skip_with_error_test --benchmark_min_time=0.01) +compile_benchmark_test(donotoptimize_test) +add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01) + compile_benchmark_test(fixture_test) add_test(fixture_test fixture_test --benchmark_min_time=0.01) diff --git a/test/donotoptimize_test.cc b/test/donotoptimize_test.cc new file mode 100644 index 0000000..e4453fb --- /dev/null +++ b/test/donotoptimize_test.cc @@ -0,0 +1,36 @@ +#include "benchmark/benchmark.h" + +#include + +namespace { +#if defined(__GNUC__) + std::uint64_t double_up(const std::uint64_t x) __attribute__ ((const)); +#endif + std::uint64_t double_up(const std::uint64_t x) { + return x * 2; + } +} + +int main(int, char*[]) { + + // this test verifies compilation of DoNotOptimize() for some types + + char buffer8[8]; + benchmark::DoNotOptimize(buffer8); + + char buffer20[20]; + benchmark::DoNotOptimize(buffer20); + + char buffer1024[1024]; + benchmark::DoNotOptimize(buffer1024); + benchmark::DoNotOptimize(&buffer1024[0]); + + int x = 123; + benchmark::DoNotOptimize(x); + benchmark::DoNotOptimize(&x); + benchmark::DoNotOptimize(x += 42); + + benchmark::DoNotOptimize(double_up(x)); + + return 0; +} -- cgit v1.2.3 From 37ab858e4b245a49805b01358655fab069474a7c Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 26 May 2016 19:44:11 +0200 Subject: initialized doubles to 0.0 --- include/benchmark/complexity.h | 12 ++++++------ src/complexity.cc | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h index f2eaefa..76318d8 100644 --- a/include/benchmark/complexity.h +++ b/include/benchmark/complexity.h @@ -48,23 +48,23 @@ enum BigO { struct LeastSq { LeastSq() : - coef(0), - rms(0), - complexity(benchmark::oNone) {} + coef(0.0), + rms(0.0), + complexity(oNone) {} double coef; double rms; - benchmark::BigO complexity; + BigO complexity; }; // Function to return an string for the calculated complexity -std::string GetBigOString(benchmark::BigO complexity); +std::string GetBigOString(BigO complexity); // Find the coefficient for the high-order term in the running time, by // minimizing the sum of squares of relative error. LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, - const benchmark::BigO complexity = benchmark::oAuto); + const BigO complexity = oAuto); } // end namespace benchmark #endif // COMPLEXITY_H_ diff --git a/src/complexity.cc b/src/complexity.cc index 8fa75b3..e723310 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -86,10 +86,10 @@ std::string GetBigOString(BigO complexity) { LeastSq CalculateLeastSq(const std::vector& n, const std::vector& time, std::function fitting_curve) { - double sigma_gn = 0; - double sigma_gn_squared = 0; - double sigma_time = 0; - double sigma_time_gn = 0; + double sigma_gn = 0.0; + double sigma_gn_squared = 0.0; + double sigma_time = 0.0; + double sigma_time_gn = 0.0; // Calculate least square fitting parameter for (size_t i = 0; i < n.size(); ++i) { @@ -106,7 +106,7 @@ LeastSq CalculateLeastSq(const std::vector& n, result.coef = sigma_time_gn / sigma_gn_squared; // Calculate RMS - double rms = 0; + double rms = 0.0; for (size_t i = 0; i < n.size(); ++i) { double fit = result.coef * fitting_curve(n[i]); rms += pow((time[i] - fit), 2); -- cgit v1.2.3 From d82f0c313133c60e3a5db5be6f7d2299cd5ffdd8 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 26 May 2016 20:57:27 +0200 Subject: added includes --- src/console_reporter.cc | 1 + src/reporter.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index da07519..bd5b403 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" +#include "benchmark/complexity.h" #include #include diff --git a/src/reporter.cc b/src/reporter.cc index 4fd0ba5..9b65c9b 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" +#include "benchmark/complexity.h" #include #include -- cgit v1.2.3 From ac3ec2ded37e4f9e4083f8b65644f7c4b60336d7 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 26 May 2016 21:16:40 +0200 Subject: moved complexity.h into src and BigO enum into benchmark_api --- include/benchmark/benchmark_api.h | 15 ++++++++- include/benchmark/complexity.h | 70 --------------------------------------- src/complexity.cc | 2 +- src/complexity.h | 58 ++++++++++++++++++++++++++++++++ src/console_reporter.cc | 2 +- src/reporter.cc | 2 +- 6 files changed, 75 insertions(+), 74 deletions(-) delete mode 100644 include/benchmark/complexity.h create mode 100644 src/complexity.h diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 9c32bac..29d0f6b 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -154,7 +154,6 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include "macros.h" -#include "complexity.h" namespace benchmark { class BenchmarkReporter; @@ -239,6 +238,20 @@ enum TimeUnit { kMillisecond }; +// BigO is passed to a benchmark in order to specify the asymptotic computational +// complexity for the benchmark. In case oAuto is selected, complexity will be +// calculated automatically to the best fit. +enum BigO { + oNone, + o1, + oN, + oNSquared, + oNCubed, + oLogN, + oNLogN, + oAuto +}; + // State is passed to a running Benchmark and contains state for the // benchmark to use. class State { diff --git a/include/benchmark/complexity.h b/include/benchmark/complexity.h deleted file mode 100644 index 76318d8..0000000 --- a/include/benchmark/complexity.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Source project : https://github.com/ismaelJimenez/cpp.leastsq -// Adapted to be used with google benchmark - -#ifndef COMPLEXITY_H_ -#define COMPLEXITY_H_ - -#include -#include - -namespace benchmark { - -// BigO is passed to a benchmark in order to specify the asymptotic computational -// complexity for the benchmark. In case oAuto is selected, complexity will be -// calculated automatically to the best fit. -enum BigO { - oNone, - o1, - oN, - oNSquared, - oNCubed, - oLogN, - oNLogN, - oAuto -}; - -// This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coeficient for the high-order term as -// interpolated from data. -// - rms : Normalized Root Mean Squared Error. -// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability -// form has been provided to MinimalLeastSq this will return -// the same value. In case BigO::oAuto has been selected, this -// parameter will return the best fitting curve detected. - -struct LeastSq { - LeastSq() : - coef(0.0), - rms(0.0), - complexity(oNone) {} - - double coef; - double rms; - BigO complexity; -}; - -// Function to return an string for the calculated complexity -std::string GetBigOString(BigO complexity); - -// Find the coefficient for the high-order term in the running time, by -// minimizing the sum of squares of relative error. -LeastSq MinimalLeastSq(const std::vector& n, - const std::vector& time, - const BigO complexity = oAuto); - -} // end namespace benchmark -#endif // COMPLEXITY_H_ diff --git a/src/complexity.cc b/src/complexity.cc index e723310..c3dd40e 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -15,7 +15,7 @@ // Source project : https://github.com/ismaelJimenez/cpp.leastsq // Adapted to be used with google benchmark -#include "benchmark/complexity.h" +#include "complexity.h" #include "check.h" #include #include diff --git a/src/complexity.h b/src/complexity.h new file mode 100644 index 0000000..9b1ae8d --- /dev/null +++ b/src/complexity.h @@ -0,0 +1,58 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Adapted to be used with google benchmark + +#ifndef COMPLEXITY_H_ +#define COMPLEXITY_H_ + +#include +#include + +#include "benchmark/benchmark_api.h" + +namespace benchmark { + +// This data structure will contain the result returned by MinimalLeastSq +// - coef : Estimated coeficient for the high-order term as +// interpolated from data. +// - rms : Normalized Root Mean Squared Error. +// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability +// form has been provided to MinimalLeastSq this will return +// the same value. In case BigO::oAuto has been selected, this +// parameter will return the best fitting curve detected. + +struct LeastSq { + LeastSq() : + coef(0.0), + rms(0.0), + complexity(oNone) {} + + double coef; + double rms; + benchmark::BigO complexity; +}; + +// Function to return an string for the calculated complexity +std::string GetBigOString(benchmark::BigO complexity); + +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error. +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + const benchmark::BigO complexity = oAuto); + +} // end namespace benchmark +#endif // COMPLEXITY_H_ diff --git a/src/console_reporter.cc b/src/console_reporter.cc index bd5b403..62df4c7 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -13,7 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" -#include "benchmark/complexity.h" +#include "complexity.h" #include #include diff --git a/src/reporter.cc b/src/reporter.cc index 9b65c9b..aa1e69a 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -13,7 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" -#include "benchmark/complexity.h" +#include "complexity.h" #include #include -- cgit v1.2.3 From 805e8baee9da3744428e3f646f321c29283c4072 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 26 May 2016 21:26:43 +0200 Subject: small refactor --- src/complexity.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/complexity.h b/src/complexity.h index 9b1ae8d..dd68362 100644 --- a/src/complexity.h +++ b/src/complexity.h @@ -42,17 +42,17 @@ struct LeastSq { double coef; double rms; - benchmark::BigO complexity; + BigO complexity; }; // Function to return an string for the calculated complexity -std::string GetBigOString(benchmark::BigO complexity); +std::string GetBigOString(BigO complexity); // Find the coefficient for the high-order term in the running time, by // minimizing the sum of squares of relative error. LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, - const benchmark::BigO complexity = oAuto); + const BigO complexity = oAuto); } // end namespace benchmark #endif // COMPLEXITY_H_ -- cgit v1.2.3 From c1c7d33279b463088550986fe6f311a3ad2faa2e Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 26 May 2016 22:39:17 +0200 Subject: added benchmar_apit to complexity.cc --- src/complexity.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/complexity.cc b/src/complexity.cc index c3dd40e..1f6fee0 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -15,6 +15,8 @@ // Source project : https://github.com/ismaelJimenez/cpp.leastsq // Adapted to be used with google benchmark +#include "benchmark/benchmark_api.h" + #include "complexity.h" #include "check.h" #include -- cgit v1.2.3 From 5686bf1b38f8aa713267097d7c1944140f71b5d3 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 27 May 2016 13:34:37 -0600 Subject: Change reporters to use a specified output and error stream. Add tests for output. (#219) * Add test for reporter output. * setup err_stream tests * Fix warnings in tests * whitespace * Fix build errors caused by super pedantic compilers * Pass streams by pointer not non-const reference --- include/benchmark/reporter.h | 32 ++++++ src/colorprint.cc | 52 ++++++++- src/colorprint.h | 10 +- src/console_reporter.cc | 52 +++++---- src/csv_reporter.cc | 53 +++++---- src/json_reporter.cc | 10 +- src/reporter.cc | 7 ++ test/CMakeLists.txt | 2 + test/reporter_output_test.cc | 264 +++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 427 insertions(+), 55 deletions(-) create mode 100644 test/reporter_output_test.cc diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 56fe1f9..ded01d2 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -14,6 +14,8 @@ #ifndef BENCHMARK_REPORTER_H_ #define BENCHMARK_REPORTER_H_ +#include +#include #include #include #include @@ -81,6 +83,10 @@ class BenchmarkReporter { bool report_rms; }; + // Construct a BenchmarkReporter with the output stream set to 'std::cout' + // and the error stream set to 'std::cerr' + BenchmarkReporter(); + // Called once for every suite of benchmarks run. // The parameter "context" contains information that the // reporter may wish to use when generating its report, for example the @@ -105,12 +111,38 @@ class BenchmarkReporter { // reported. virtual void Finalize(); + // REQUIRES: The object referenced by 'out' is valid for the lifetime + // of the reporter. + void SetOutputStream(std::ostream* out) { + assert(out); + output_stream_ = out; + } + + // REQUIRES: The object referenced by 'err' is valid for the lifetime + // of the reporter. + void SetErrorStream(std::ostream* err) { + assert(err); + error_stream_ = err; + } + + std::ostream& GetOutputStream() const { + return *output_stream_; + } + + std::ostream& GetErrorStream() const { + return *error_stream_; + } + virtual ~BenchmarkReporter(); protected: static void ComputeStats(const std::vector& reports, Run* mean, Run* stddev); static void ComputeBigO(const std::vector& reports, Run* bigO, Run* rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); + +private: + std::ostream* output_stream_; + std::ostream* error_stream_; }; // Simple reporter that outputs benchmark data to the console. This is the diff --git a/src/colorprint.cc b/src/colorprint.cc index 81f917b..efb8626 100644 --- a/src/colorprint.cc +++ b/src/colorprint.cc @@ -16,8 +16,12 @@ #include #include +#include +#include +#include #include "commandlineflags.h" +#include "check.h" #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS @@ -74,14 +78,51 @@ PlatformColorCode GetPlatformColorCode(LogColor color) { }; #endif } + } // end namespace -void ColorPrintf(LogColor color, const char* fmt, ...) { +std::string FormatString(const char *msg, va_list args) { + // we might need a second shot at this, so pre-emptivly make a copy + va_list args_cp; + va_copy(args_cp, args); + + std::size_t size = 256; + char local_buff[256]; + auto ret = std::vsnprintf(local_buff, size, msg, args_cp); + + va_end(args_cp); + + // currently there is no error handling for failure, so this is hack. + CHECK(ret >= 0); + + if (ret == 0) // handle empty expansion + return {}; + else if (static_cast(ret) < size) + return local_buff; + else { + // we did not provide a long enough buffer on our first attempt. + size = (size_t)ret + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + ret = std::vsnprintf(buff.get(), size, msg, args); + CHECK(ret > 0 && ((size_t)ret) < size); + return buff.get(); + } +} + +std::string FormatString(const char *msg, ...) { + va_list args; + va_start(args, msg); + auto tmp = FormatString(msg, args); + va_end(args); + return tmp; +} + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...) { va_list args; va_start(args, fmt); if (!FLAGS_color_print) { - vprintf(fmt, args); + out << FormatString(fmt, args); va_end(args); return; } @@ -107,10 +148,11 @@ void ColorPrintf(LogColor color, const char* fmt, ...) { SetConsoleTextAttribute(stdout_handle, old_color_attrs); #else const char* color_code = GetPlatformColorCode(color); - if (color_code) fprintf(stdout, "\033[0;3%sm", color_code); - vprintf(fmt, args); - printf("\033[m"); // Resets the terminal to default. + if (color_code) out << FormatString("\033[0;3%sm", color_code); + out << FormatString(fmt, args) << "\033[m"; #endif + va_end(args); } + } // end namespace benchmark diff --git a/src/colorprint.h b/src/colorprint.h index 54d1f66..2b3c082 100644 --- a/src/colorprint.h +++ b/src/colorprint.h @@ -1,6 +1,10 @@ #ifndef BENCHMARK_COLORPRINT_H_ #define BENCHMARK_COLORPRINT_H_ +#include +#include +#include + namespace benchmark { enum LogColor { COLOR_DEFAULT, @@ -13,7 +17,11 @@ enum LogColor { COLOR_WHITE }; -void ColorPrintf(LogColor color, const char* fmt, ...); +std::string FormatString(const char* msg, va_list args); +std::string FormatString(const char* msg, ...); + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...); + } // end namespace benchmark #endif // BENCHMARK_COLORPRINT_H_ diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 5dd98f6..6e39ad9 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -25,6 +25,8 @@ #include "check.h" #include "colorprint.h" +#include "commandlineflags.h" +#include "internal_macros.h" #include "string_util.h" #include "walltime.h" @@ -33,26 +35,36 @@ namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; - std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu + auto& Out = GetOutputStream(); + auto& Err = GetErrorStream(); + +#ifdef BENCHMARK_OS_WINDOWS + if (FLAGS_color_print && &Out != &std::cout) { + Err << "Color printing is only supported for stdout on windows. " + "Disabling color printing\n"; + FLAGS_color_print = false; + } +#endif + + Err << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; - std::cerr << LocalDateTimeString() << "\n"; + Err << LocalDateTimeString() << "\n"; if (context.cpu_scaling_enabled) { - std::cerr << "***WARNING*** CPU scaling is enabled, the benchmark " + Err << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " "overhead.\n"; } #ifndef NDEBUG - std::cerr << "***WARNING*** Library was built as DEBUG. Timings may be " + Err << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; #endif - - int output_width = fprintf(stdout, "%-*s %13s %13s %10s\n", + std::string str = FormatString("%-*s %13s %13s %10s\n", static_cast(name_field_width_), "Benchmark", "Time", "CPU", "Iterations"); - std::cout << std::string(output_width - 1, '-') << "\n"; + Out << str << std::string(str.length() - 1, '-') << "\n"; return true; } @@ -101,15 +113,17 @@ void ConsoleReporter::ReportComplexity(const std::vector & complexity_repor } void ConsoleReporter::PrintRunData(const Run& result) { + auto& Out = GetOutputStream(); + auto name_color = (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; - ColorPrintf(name_color, "%-*s ", name_field_width_, + ColorPrintf(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if (result.error_occurred) { - ColorPrintf(COLOR_RED, "ERROR OCCURRED: \'%s\'", + ColorPrintf(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", result.error_message.c_str()); - ColorPrintf(COLOR_DEFAULT, "\n"); + ColorPrintf(Out, COLOR_DEFAULT, "\n"); return; } // Format bytes per second @@ -131,24 +145,24 @@ void ConsoleReporter::PrintRunData(const Run& result) { if(result.report_big_o) { std::string big_o = result.report_big_o ? GetBigOString(result.complexity) : ""; - ColorPrintf(COLOR_YELLOW, "%10.4f %s %10.4f %s ", + ColorPrintf(Out, COLOR_YELLOW, "%10.4f %s %10.4f %s ", result.real_accumulated_time * multiplier, big_o.c_str(), result.cpu_accumulated_time * multiplier, big_o.c_str()); } else if(result.report_rms) { - ColorPrintf(COLOR_YELLOW, "%10.0f %% %10.0f %% ", + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", result.real_accumulated_time * multiplier * 100, result.cpu_accumulated_time * multiplier * 100); } else if (result.iterations == 0) { - ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, timeLabel, result.cpu_accumulated_time * multiplier, timeLabel); } else { - ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", (result.real_accumulated_time * multiplier) / (static_cast(result.iterations)), timeLabel, @@ -158,22 +172,22 @@ void ConsoleReporter::PrintRunData(const Run& result) { } if(!result.report_big_o && !result.report_rms) { - ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); + ColorPrintf(Out, COLOR_CYAN, "%10lld", result.iterations); } if (!rate.empty()) { - ColorPrintf(COLOR_DEFAULT, " %*s", 13, rate.c_str()); + ColorPrintf(Out, COLOR_DEFAULT, " %*s", 13, rate.c_str()); } if (!items.empty()) { - ColorPrintf(COLOR_DEFAULT, " %*s", 18, items.c_str()); + ColorPrintf(Out, COLOR_DEFAULT, " %*s", 18, items.c_str()); } if (!result.report_label.empty()) { - ColorPrintf(COLOR_DEFAULT, " %s", result.report_label.c_str()); + ColorPrintf(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); } - ColorPrintf(COLOR_DEFAULT, "\n"); + ColorPrintf(Out, COLOR_DEFAULT, "\n"); } } // end namespace benchmark diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index ac4bd3d..36f149b 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -44,27 +44,30 @@ std::vector elements = { } bool CSVReporter::ReportContext(const Context& context) { - std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu + std::ostream& Err = GetErrorStream(); + std::ostream& Out = GetOutputStream(); + + Err << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; - std::cerr << LocalDateTimeString() << "\n"; + Err << LocalDateTimeString() << "\n"; if (context.cpu_scaling_enabled) { - std::cerr << "***WARNING*** CPU scaling is enabled, the benchmark " + Err << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " "overhead.\n"; } #ifndef NDEBUG - std::cerr << "***WARNING*** Library was built as DEBUG. Timings may be " + Err << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; #endif for (auto B = elements.begin(); B != elements.end(); ) { - std::cout << *B++; + Out << *B++; if (B != elements.end()) - std::cout << ","; + Out << ","; } - std::cout << "\n"; + Out << "\n"; return true; } @@ -106,19 +109,19 @@ void CSVReporter::ReportComplexity(const std::vector& complexity_reports) { } void CSVReporter::PrintRunData(const Run & run) { - + std::ostream& Out = GetOutputStream(); // Field with embedded double-quote characters must be doubled and the field // delimited with double-quotes. std::string name = run.benchmark_name; ReplaceAll(&name, "\"", "\"\""); - std::cout << '"' << name << "\","; + Out << '"' << name << "\","; if (run.error_occurred) { - std::cout << std::string(elements.size() - 3, ','); - std::cout << "true,"; + Out << std::string(elements.size() - 3, ','); + Out << "true,"; std::string msg = run.error_message; ReplaceAll(&msg, "\"", "\"\""); - std::cout << '"' << msg << "\"\n"; + Out << '"' << msg << "\"\n"; return; } @@ -135,36 +138,36 @@ void CSVReporter::PrintRunData(const Run & run) { // Do not print iteration on bigO and RMS report if(!run.report_big_o && !run.report_rms) { - std::cout << run.iterations; + Out << run.iterations; } - std::cout << ","; + Out << ","; - std::cout << real_time << ","; - std::cout << cpu_time << ","; + Out << real_time << ","; + Out << cpu_time << ","; // Do not print timeLabel on RMS report if(!run.report_rms) { - std::cout << timeLabel; + Out << timeLabel; } - std::cout << ","; + Out << ","; if (run.bytes_per_second > 0.0) { - std::cout << run.bytes_per_second; + Out << run.bytes_per_second; } - std::cout << ","; + Out << ","; if (run.items_per_second > 0.0) { - std::cout << run.items_per_second; + Out << run.items_per_second; } - std::cout << ","; + Out << ","; if (!run.report_label.empty()) { // Field with embedded double-quote characters must be doubled and the field // delimited with double-quotes. std::string label = run.report_label; ReplaceAll(&label, "\"", "\"\""); - std::cout << "\"" << label << "\""; + Out << "\"" << label << "\""; } - std::cout << ",,"; // for error_occurred and error_message - std::cout << '\n'; + Out << ",,"; // for error_occurred and error_message + Out << '\n'; } } // end namespace benchmark diff --git a/src/json_reporter.cc b/src/json_reporter.cc index cab527c..d3effe1 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -53,7 +53,7 @@ int64_t RoundDouble(double v) { } // end namespace bool JSONReporter::ReportContext(const Context& context) { - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); out << "{\n"; std::string inner_indent(2, ' '); @@ -92,7 +92,7 @@ void JSONReporter::ReportRuns(std::vector const& reports) { return; } std::string indent(4, ' '); - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); if (!first_report_) { out << ",\n"; } @@ -128,7 +128,7 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) } std::string indent(4, ' '); - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); if (!first_report_) { out << ",\n"; } @@ -148,7 +148,7 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) void JSONReporter::Finalize() { // Close the list of benchmarks and the top level object. - std::cout << "\n ]\n}\n"; + GetOutputStream() << "\n ]\n}\n"; } void JSONReporter::PrintRunData(Run const& run) { @@ -164,7 +164,7 @@ void JSONReporter::PrintRunData(Run const& run) { } std::string indent(6, ' '); - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name) << ",\n"; diff --git a/src/reporter.cc b/src/reporter.cc index d0a80e6..5b303f7 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -16,6 +16,8 @@ #include "complexity.h" #include + +#include #include #include @@ -24,6 +26,11 @@ namespace benchmark { +BenchmarkReporter::BenchmarkReporter() + : output_stream_(&std::cout), error_stream_(&std::cerr) +{ +} + void BenchmarkReporter::ComputeStats( const std::vector& reports, Run* mean_data, Run* stddev_data) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 247c630..aeb720a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,6 +48,8 @@ add_test(fixture_test fixture_test --benchmark_min_time=0.01) compile_benchmark_test(map_test) add_test(map_test map_test --benchmark_min_time=0.01) +compile_benchmark_test(reporter_output_test) +add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc new file mode 100644 index 0000000..acfbf68 --- /dev/null +++ b/test/reporter_output_test.cc @@ -0,0 +1,264 @@ + +#undef NDEBUG +#include "benchmark/benchmark.h" +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "../src/re.h" // NOTE: re.h is for internal use only +#include +#include +#include +#include +#include +#include + +namespace { + +// ========================================================================= // +// -------------------------- Testing Case --------------------------------- // +// ========================================================================= // + +enum MatchRules { + MR_Default, // Skip non-matching lines until a match is found. + MR_Next // Match must occur on the next line. +}; + +struct TestCase { + std::string regex; + int match_rule; + + TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {} + + void Check(std::stringstream& remaining_output) const { + benchmark::Regex r; + std::string err_str; + r.Init(regex, &err_str); + CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\"" + << " got Error: " << err_str; + + std::string line; + while (remaining_output.eof() == false) { + CHECK(remaining_output.good()); + std::getline(remaining_output, line); + if (r.Match(line)) return; + CHECK(match_rule != MR_Next) << "Expected line \"" << line + << "\" to match regex \"" << regex << "\""; + } + + CHECK(remaining_output.eof() == false) + << "End of output reached before match for regex \"" << regex + << "\" was found"; + } +}; + +std::vector ConsoleOutputTests; +std::vector JSONOutputTests; +std::vector CSVOutputTests; + +std::vector ConsoleErrorTests; +std::vector JSONErrorTests; +std::vector CSVErrorTests; + +// ========================================================================= // +// -------------------------- Test Helpers --------------------------------- // +// ========================================================================= // + +class TestReporter : public benchmark::BenchmarkReporter { +public: + TestReporter(std::vector reps) + : reporters_(reps) {} + + virtual bool ReportContext(const Context& context) { + bool last_ret = false; + bool first = true; + for (auto rep : reporters_) { + bool new_ret = rep->ReportContext(context); + CHECK(first || new_ret == last_ret) + << "Reports return different values for ReportContext"; + first = false; + last_ret = new_ret; + } + return last_ret; + } + + virtual void ReportRuns(const std::vector& report) { + for (auto rep : reporters_) + rep->ReportRuns(report); + } + + virtual void ReportComplexity(const std::vector& complexity_reports) { + for (auto rep : reporters_) + rep->ReportComplexity(complexity_reports); + } + + virtual void Finalize() { + for (auto rep : reporters_) + rep->Finalize(); + } + +private: + std::vector reporters_; +}; + + +#define CONCAT2(x, y) x##y +#define CONCAT(x, y) CONCAT2(x, y) + +#define ADD_CASES(...) \ + int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +int AddCases(std::vector* out, std::initializer_list const& v) { + for (auto const& TC : v) + out->push_back(TC); + return 0; +} + +template +std::string join(First f) { return f; } + +template +std::string join(First f, Args&&... args) { + return std::string(std::move(f)) + "[ ]+" + join(std::forward(args)...); +} + +std::string dec_re = "[0-9]+\\.[0-9]+"; + +} // end namespace + +// ========================================================================= // +// ---------------------- Testing Prologue Output -------------------------- // +// ========================================================================= // + +ADD_CASES(&ConsoleOutputTests, { + {join("^Benchmark", "Time", "CPU", "Iterations$"), MR_Next}, + {"^[-]+$", MR_Next} +}); +ADD_CASES(&CSVOutputTests, { + {"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,items_per_second," + "label,error_occurred,error_message"} +}); + +// ========================================================================= // +// ------------------------ Testing Basic Output --------------------------- // +// ========================================================================= // + +void BM_basic(benchmark::State& state) { + while (state.KeepRunning()) {} +} +BENCHMARK(BM_basic); + +ADD_CASES(&ConsoleOutputTests, { + {"^BM_basic[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"} +}); +ADD_CASES(&JSONOutputTests, { + {"\"name\": \"BM_basic\",$"}, + {"\"iterations\": [0-9]+,$", MR_Next}, + {"\"real_time\": [0-9],$", MR_Next}, + {"\"cpu_time\": [0-9],$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next} +}); +ADD_CASES(&CSVOutputTests, { + {"^\"BM_basic\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} +}); + +// ========================================================================= // +// ------------------------ Testing Error Output --------------------------- // +// ========================================================================= // + +void BM_error(benchmark::State& state) { + state.SkipWithError("message"); + while(state.KeepRunning()) {} +} +BENCHMARK(BM_error); +ADD_CASES(&ConsoleOutputTests, { + {"^BM_error[ ]+ERROR OCCURRED: 'message'$"} +}); +ADD_CASES(&JSONOutputTests, { + {"\"name\": \"BM_error\",$"}, + {"\"error_occurred\": true,$", MR_Next}, + {"\"error_message\": \"message\",$", MR_Next} +}); + +ADD_CASES(&CSVOutputTests, { + {"^\"BM_error\",,,,,,,,true,\"message\"$"} +}); + + +// ========================================================================= // +// ----------------------- Testing Complexity Output ----------------------- // +// ========================================================================= // + +void BM_Complexity_O1(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.SetComplexityN(state.range_x()); +} +BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); + +std::string bigOStr = "[0-9]+\\.[0-9]+ \\* [0-9]+"; + +ADD_CASES(&ConsoleOutputTests, { + {join("^BM_Complexity_O1_BigO", bigOStr, bigOStr) + "[ ]*$"}, + {join("^BM_Complexity_O1_RMS", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"} +}); + + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + + +int main(int argc, char* argv[]) { + // Add --color_print=false to argv since we don't want to match color codes. + char new_arg[64]; + char* new_argv[64]; + std::copy(argv, argv + argc, new_argv); + new_argv[argc++] = std::strcpy(new_arg, "--color_print=false"); + benchmark::Initialize(&argc, new_argv); + + benchmark::ConsoleReporter CR; + benchmark::JSONReporter JR; + benchmark::CSVReporter CSVR; + struct ReporterTest { + const char* name; + std::vector& output_cases; + std::vector& error_cases; + benchmark::BenchmarkReporter& reporter; + std::stringstream out_stream; + std::stringstream err_stream; + + ReporterTest(const char* n, + std::vector& out_tc, + std::vector& err_tc, + benchmark::BenchmarkReporter& br) + : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { + reporter.SetOutputStream(&out_stream); + reporter.SetErrorStream(&err_stream); + } + } TestCases[] = { + {"ConsoleReporter", ConsoleOutputTests, ConsoleErrorTests, CR}, + {"JSONReporter", JSONOutputTests, JSONErrorTests, JR}, + {"CSVReporter", CSVOutputTests, CSVErrorTests, CSVR} + }; + + // Create the test reporter and run the benchmarks. + std::cout << "Running benchmarks...\n"; + TestReporter test_rep({&CR, &JR, &CSVR}); + benchmark::RunSpecifiedBenchmarks(&test_rep); + + for (auto& rep_test : TestCases) { + std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string banner(msg.size() - 1, '-'); + std::cout << banner << msg << banner << "\n"; + + std::cerr << rep_test.err_stream.str(); + std::cout << rep_test.out_stream.str(); + + for (const auto& TC : rep_test.error_cases) + TC.Check(rep_test.err_stream); + for (const auto& TC : rep_test.output_cases) + TC.Check(rep_test.out_stream); + + std::cout << "\n"; + } + return 0; +} -- cgit v1.2.3 From 238e558fdb6f00a2e3eb75d6c353030f8a510f8c Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 27 May 2016 13:37:10 -0600 Subject: Allow benchmarks to take arbitrary arguments. (#221) * Add lambda benchmarks * Remove lambda capture since the lambda is not at a block scope * Remove LambdaBenchmark helper since FunctionBenchmark can be used with non-capturing lambas * Add lambda benchmarks * Remove lambda capture since the lambda is not at a block scope * Remove LambdaBenchmark helper since FunctionBenchmark can be used with non-capturing lambas * Add more docs for BENCHMARK_CAPTURE. * Fix use of misnamed parameter * Guard BENCHMARK_CAPTURE tests against non-c++11 compilers * Move tests out of basic_test.cc --- README.md | 20 ++++++++++++++++++++ include/benchmark/benchmark_api.h | 22 ++++++++++++++++++++++ test/benchmark_test.cc | 18 ++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/README.md b/README.md index 377fe10..5be5153 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,26 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` +## Passing arbitrary arguments to a benchmark +In C++11 it is possible to define a benchmark that takes an arbitrary number +of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` +macro creates a benchmark that invokes `func` with the `benchmark::State` as +the first argument followed by the specified `args...`. +The `test_case_name` is appended to the name of the benchmark and +should describe the values passed. + +```c++ +template ` +void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { + [...] +} +// Registers a benchmark named "BM_takes_args/int_string_test` that passes +// the specified values to `extra_args`. +BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +``` +Note that elements of `...args` may refer to global variables. Users should +avoid modifying global state inside of a benchmark. + ### Multithreaded benchmarks In a multithreaded test (benchmark invoked by multiple threads simultaneously), it is guaranteed that none of the threads will start until all have called diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 9f3be61..e705d75 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -650,6 +650,28 @@ protected: #define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ BENCHMARK(n)->RangePair((l1), (h1), (l2), (h2)) +#if __cplusplus >= 201103L + +// Register a benchmark which invokes the function specified by `func` +// with the additional arguments specified by `...`. +// +// For example: +// +// template ` +// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { +// [...] +//} +// /* Registers a benchmark named "BM_takes_args/int_string_test` */ +// BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #func "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + +#endif // __cplusplus >= 11 + // This will register a benchmark for a templatized function. For example: // // template diff --git a/test/benchmark_test.cc b/test/benchmark_test.cc index 252602a..66f5956 100644 --- a/test/benchmark_test.cc +++ b/test/benchmark_test.cc @@ -16,6 +16,7 @@ #include #include #include +#include #if defined(__GNUC__) # define BENCHMARK_NOINLINE __attribute__((noinline)) @@ -202,5 +203,22 @@ static void BM_ManualTiming(benchmark::State& state) { BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime(); BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime(); +#if __cplusplus >= 201103L + +template +void BM_with_args(benchmark::State& state, Args&&...) { + while (state.KeepRunning()) {} +} +BENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44); +BENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, + std::string("abc"), std::pair(42, 3.8)); + +void BM_non_template_args(benchmark::State& state, int, double) { + while(state.KeepRunning()) {} +} +BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); + +#endif // __cplusplus >= 201103L + BENCHMARK_MAIN() -- cgit v1.2.3 From 1b263fe6d906bb0854b84247f1b395bbacd3b88e Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 27 May 2016 16:45:25 -0600 Subject: Cleanup reporters (#226) * Move ComputeStats call out of the reporters * Cleanup adjusted time calculations in reporters * Move ComputeBigO call out of reporters * Remove ReportComplexity interface using ReportRuns instead * Factor out reporting of basic context information * Attempt to fix GCC 4.6 build errors * Move ComputeStats to complexity.cc --- include/benchmark/benchmark_api.h | 24 +++++++ include/benchmark/reporter.h | 46 ++++++------ src/benchmark.cc | 12 +++- src/complexity.cc | 134 +++++++++++++++++++++++++++++++++- src/complexity.h | 12 ++++ src/console_reporter.cc | 95 ++++--------------------- src/csv_reporter.cc | 72 ++----------------- src/json_reporter.cc | 58 ++------------- src/reporter.cc | 146 ++++++++------------------------------ test/reporter_output_test.cc | 5 -- 10 files changed, 257 insertions(+), 347 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index e705d75..5929c66 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -236,6 +236,30 @@ enum TimeUnit { kMillisecond }; +inline const char* GetTimeUnitString(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + default: + return "ns"; + } +} + +inline double GetTimeUnitMultiplier(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + default: + return 1e9; + } +} + // BigO is passed to a benchmark in order to specify the asymptotic computational // complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index ded01d2..da53faa 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -24,8 +24,6 @@ namespace benchmark { -typedef std::pair TimeUnitMultiplier; - // Interface for custom benchmark result printers. // By default, benchmark reports are printed to stdout. However an application // can control the destination of the reports by calling @@ -67,6 +65,18 @@ class BenchmarkReporter { double real_accumulated_time; double cpu_accumulated_time; + // Return a value representing the real time per iteration in the unit + // specified by 'time_unit'. + // NOTE: If 'iterations' is zero the returned value represents the + // accumulated time. + double GetAdjustedRealTime() const; + + // Return a value representing the cpu time per iteration in the unit + // specified by 'time_unit'. + // NOTE: If 'iterations' is zero the returned value represents the + // accumulated time. + double GetAdjustedCPUTime() const; + // Zero if not set by benchmark. double bytes_per_second; double items_per_second; @@ -96,20 +106,17 @@ class BenchmarkReporter { virtual bool ReportContext(const Context& context) = 0; // Called once for each group of benchmark runs, gives information about - // cpu-time and heap memory usage during the benchmark run. - // Note that all the grouped benchmark runs should refer to the same - // benchmark, thus have the same name. + // cpu-time and heap memory usage during the benchmark run. If the group + // of runs contained more than two entries then 'report' contains additional + // elements representing the mean and standard deviation of those runs. + // Additionally if this group of runs was the last in a family of benchmarks + // 'reports' contains additional entries representing the asymptotic + // complexity and RMS of that benchmark family. virtual void ReportRuns(const std::vector& report) = 0; - // Called once at the last benchmark in a family of benchmarks, gives information - // about asymptotic complexity and RMS. - // Note that all the benchmark runs in a range should refer to the same benchmark, - // thus have the same name. - virtual void ReportComplexity(const std::vector& complexity_reports) = 0; - // Called once and only once after ever group of benchmarks is run and // reported. - virtual void Finalize(); + virtual void Finalize() {} // REQUIRES: The object referenced by 'out' is valid for the lifetime // of the reporter. @@ -134,11 +141,11 @@ class BenchmarkReporter { } virtual ~BenchmarkReporter(); -protected: - static void ComputeStats(const std::vector& reports, - Run* mean, Run* stddev); - static void ComputeBigO(const std::vector& reports, Run* bigO, Run* rms); - static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); + + // Write a human readable string to 'out' representing the specified + // 'context'. + // REQUIRES: 'out' is non-null. + static void PrintBasicContext(std::ostream* out, Context const& context); private: std::ostream* output_stream_; @@ -151,9 +158,8 @@ class ConsoleReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); - virtual void ReportComplexity(const std::vector& complexity_reports); - protected: +protected: virtual void PrintRunData(const Run& report); size_t name_field_width_; @@ -164,7 +170,6 @@ public: JSONReporter() : first_report_(true) {} virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); - virtual void ReportComplexity(const std::vector& complexity_reports); virtual void Finalize(); private: @@ -177,7 +182,6 @@ class CSVReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); - virtual void ReportComplexity(const std::vector& complexity_reports); private: void PrintRunData(const Run& report); diff --git a/src/benchmark.cc b/src/benchmark.cc index f302356..d86705e 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -33,6 +33,7 @@ #include "check.h" #include "commandlineflags.h" +#include "complexity.h" #include "log.h" #include "mutex.h" #include "re.h" @@ -717,7 +718,6 @@ void FunctionBenchmark::Run(State& st) { namespace { - // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into *total. void RunInThread(const benchmark::internal::Benchmark::Instance* b, @@ -876,13 +876,19 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, iters = static_cast(next_iters + 0.5); } } - br->ReportRuns(reports); + std::vector additional_run_stats = ComputeStats(reports); + reports.insert(reports.end(), additional_run_stats.begin(), + additional_run_stats.end()); if((b.complexity != oNone) && b.last_benchmark_instance) { - br->ReportComplexity(complexity_reports); + additional_run_stats = ComputeBigO(complexity_reports); + reports.insert(reports.end(), additional_run_stats.begin(), + additional_run_stats.end()); complexity_reports.clear(); } + br->ReportRuns(reports); + if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); diff --git a/src/complexity.cc b/src/complexity.cc index 1f6fee0..3e42f5d 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -19,7 +19,9 @@ #include "complexity.h" #include "check.h" -#include +#include "stat.h" +#include +#include #include namespace benchmark { @@ -161,4 +163,134 @@ LeastSq MinimalLeastSq(const std::vector& n, return best_fit; } +std::vector ComputeStats( + const std::vector& reports) +{ + typedef BenchmarkReporter::Run Run; + std::vector results; + + auto error_count = std::count_if( + reports.begin(), reports.end(), + [](Run const& run) {return run.error_occurred;}); + + if (reports.size() - error_count < 2) { + // We don't report aggregated data if there was a single run. + return results; + } + // Accumulators. + Stat1_d real_accumulated_time_stat; + Stat1_d cpu_accumulated_time_stat; + Stat1_d bytes_per_second_stat; + Stat1_d items_per_second_stat; + // All repetitions should be run with the same number of iterations so we + // can take this information from the first benchmark. + int64_t const run_iterations = reports.front().iterations; + + // Populate the accumulators. + for (Run const& run : reports) { + CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); + CHECK_EQ(run_iterations, run.iterations); + if (run.error_occurred) + continue; + real_accumulated_time_stat += + Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); + cpu_accumulated_time_stat += + Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); + items_per_second_stat += Stat1_d(run.items_per_second, run.iterations); + bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations); + } + + // Get the data from the accumulator to BenchmarkReporter::Run's. + Run mean_data; + mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; + mean_data.iterations = run_iterations; + mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * + run_iterations; + mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * + run_iterations; + mean_data.bytes_per_second = bytes_per_second_stat.Mean(); + mean_data.items_per_second = items_per_second_stat.Mean(); + + // Only add label to mean/stddev if it is same for all runs + mean_data.report_label = reports[0].report_label; + for (std::size_t i = 1; i < reports.size(); i++) { + if (reports[i].report_label != reports[0].report_label) { + mean_data.report_label = ""; + break; + } + } + + Run stddev_data; + stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; + stddev_data.report_label = mean_data.report_label; + stddev_data.iterations = 0; + stddev_data.real_accumulated_time = + real_accumulated_time_stat.StdDev(); + stddev_data.cpu_accumulated_time = + cpu_accumulated_time_stat.StdDev(); + stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); + stddev_data.items_per_second = items_per_second_stat.StdDev(); + + results.push_back(mean_data); + results.push_back(stddev_data); + return results; +} + +std::vector ComputeBigO( + const std::vector& reports) +{ + typedef BenchmarkReporter::Run Run; + std::vector results; + + if (reports.size() < 2) return results; + + // Accumulators. + std::vector n; + std::vector real_time; + std::vector cpu_time; + + // Populate the accumulators. + for (const Run& run : reports) { + n.push_back(run.complexity_n); + real_time.push_back(run.real_accumulated_time/run.iterations); + cpu_time.push_back(run.cpu_accumulated_time/run.iterations); + } + + LeastSq result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); + + // result_cpu.complexity is passed as parameter to result_real because in case + // reports[0].complexity is oAuto, the noise on the measured data could make + // the best fit function of Cpu and Real differ. In order to solve this, we + // take the best fitting function for the Cpu, and apply it to Real data. + LeastSq result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + + std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); + + // Get the data from the accumulator to BenchmarkReporter::Run's. + Run big_o; + big_o.benchmark_name = benchmark_name + "_BigO"; + big_o.iterations = 0; + big_o.real_accumulated_time = result_real.coef; + big_o.cpu_accumulated_time = result_cpu.coef; + big_o.report_big_o = true; + big_o.complexity = result_cpu.complexity; + + double multiplier = GetTimeUnitMultiplier(reports[0].time_unit); + + // Only add label to mean/stddev if it is same for all runs + Run rms; + big_o.report_label = reports[0].report_label; + rms.benchmark_name = benchmark_name + "_RMS"; + rms.report_label = big_o.report_label; + rms.iterations = 0; + rms.real_accumulated_time = result_real.rms / multiplier; + rms.cpu_accumulated_time = result_cpu.rms / multiplier; + rms.report_rms = true; + rms.complexity = result_cpu.complexity; + + results.push_back(big_o); + results.push_back(rms); + return results; +} + } // end namespace benchmark diff --git a/src/complexity.h b/src/complexity.h index dd68362..be095a9 100644 --- a/src/complexity.h +++ b/src/complexity.h @@ -22,9 +22,21 @@ #include #include "benchmark/benchmark_api.h" +#include "benchmark/reporter.h" namespace benchmark { +// Return a vector containing the mean and standard devation information for +// the specified list of reports. If 'reports' contains less than two +// non-errored runs an empty vector is returned +std::vector ComputeStats( + const std::vector& reports); + +// Return a vector containing the bigO and RMS information for the specified +// list of reports. If 'reports.size() < 2' an empty vector is returned. +std::vector ComputeBigO( + const std::vector& reports); + // This data structure will contain the result returned by MinimalLeastSq // - coef : Estimated coeficient for the high-order term as // interpolated from data. diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 6e39ad9..87b3216 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -35,81 +35,26 @@ namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; - auto& Out = GetOutputStream(); - auto& Err = GetErrorStream(); + PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if (FLAGS_color_print && &Out != &std::cout) { - Err << "Color printing is only supported for stdout on windows. " - "Disabling color printing\n"; + if (FLAGS_color_print && &Out != &GetOutputStream()) { + GetErrorString() << "Color printing is only supported for stdout on windows." + " Disabling color printing\n"; FLAGS_color_print = false; } #endif - - Err << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu - << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; - - Err << LocalDateTimeString() << "\n"; - - if (context.cpu_scaling_enabled) { - Err << "***WARNING*** CPU scaling is enabled, the benchmark " - "real time measurements may be noisy and will incur extra " - "overhead.\n"; - } - -#ifndef NDEBUG - Err << "***WARNING*** Library was built as DEBUG. Timings may be " - "affected.\n"; -#endif std::string str = FormatString("%-*s %13s %13s %10s\n", static_cast(name_field_width_), "Benchmark", "Time", "CPU", "Iterations"); - Out << str << std::string(str.length() - 1, '-') << "\n"; + GetOutputStream() << str << std::string(str.length() - 1, '-') << "\n"; return true; } void ConsoleReporter::ReportRuns(const std::vector& reports) { - if (reports.empty()) { - return; - } - - for (Run const& run : reports) { - CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); + for (const auto& run : reports) PrintRunData(run); - } - - auto error_count = std::count_if( - reports.begin(), reports.end(), - [](Run const& run) {return run.error_occurred;}); - - if (reports.size() - error_count < 2) { - // We don't report aggregated data if there was a single run. - return; - } - - Run mean_data; - Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); - - // Output using PrintRun. - PrintRunData(mean_data); - PrintRunData(stddev_data); -} - -void ConsoleReporter::ReportComplexity(const std::vector & complexity_reports) { - if (complexity_reports.size() < 2) { - // We don't report asymptotic complexity data if there was a single run. - return; - } - - Run big_o_data; - Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); - - // Output using PrintRun. - PrintRunData(big_o_data); - PrintRunData(rms_data); } void ConsoleReporter::PrintRunData(const Run& result) { @@ -139,36 +84,20 @@ void ConsoleReporter::PrintRunData(const Run& result) { " items/s"); } - double multiplier; - const char* timeLabel; - std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(result.time_unit); + const double real_time = result.GetAdjustedRealTime(); + const double cpu_time = result.GetAdjustedCPUTime(); if(result.report_big_o) { std::string big_o = result.report_big_o ? GetBigOString(result.complexity) : ""; ColorPrintf(Out, COLOR_YELLOW, "%10.4f %s %10.4f %s ", - result.real_accumulated_time * multiplier, - big_o.c_str(), - result.cpu_accumulated_time * multiplier, - big_o.c_str()); + real_time, big_o.c_str(), cpu_time, big_o.c_str()); } else if(result.report_rms) { ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", - result.real_accumulated_time * multiplier * 100, - result.cpu_accumulated_time * multiplier * 100); - } else if (result.iterations == 0) { - - ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", - result.real_accumulated_time * multiplier, - timeLabel, - result.cpu_accumulated_time * multiplier, - timeLabel); + real_time * 100, cpu_time * 100); } else { + const char* timeLabel = GetTimeUnitString(result.time_unit); ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", - (result.real_accumulated_time * multiplier) / - (static_cast(result.iterations)), - timeLabel, - (result.cpu_accumulated_time * multiplier) / - (static_cast(result.iterations)), - timeLabel); + real_time, timeLabel, cpu_time, timeLabel); } if(!result.report_big_o && !result.report_rms) { diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 36f149b..5c18c9e 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -44,24 +44,9 @@ std::vector elements = { } bool CSVReporter::ReportContext(const Context& context) { - std::ostream& Err = GetErrorStream(); - std::ostream& Out = GetOutputStream(); - - Err << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu - << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; - - Err << LocalDateTimeString() << "\n"; - - if (context.cpu_scaling_enabled) { - Err << "***WARNING*** CPU scaling is enabled, the benchmark " - "real time measurements may be noisy and will incur extra " - "overhead.\n"; - } + PrintBasicContext(&GetErrorStream(), context); -#ifndef NDEBUG - Err << "***WARNING*** Library was built as DEBUG. Timings may be " - "affected.\n"; -#endif + std::ostream& Out = GetOutputStream(); for (auto B = elements.begin(); B != elements.end(); ) { Out << *B++; if (B != elements.end()) @@ -72,40 +57,8 @@ bool CSVReporter::ReportContext(const Context& context) { } void CSVReporter::ReportRuns(const std::vector & reports) { - if (reports.empty()) { - return; - } - - auto error_count = std::count_if( - reports.begin(), reports.end(), - [](Run const& run) {return run.error_occurred;}); - - std::vector reports_cp = reports; - if (reports.size() - error_count >= 2) { - Run mean_data; - Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); - reports_cp.push_back(mean_data); - reports_cp.push_back(stddev_data); - } - for (auto it = reports_cp.begin(); it != reports_cp.end(); ++it) { - PrintRunData(*it); - } -} - -void CSVReporter::ReportComplexity(const std::vector& complexity_reports) { - if (complexity_reports.size() < 2) { - // We don't report asymptotic complexity data if there was a single run. - return; - } - - Run big_o_data; - Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); - - // Output using PrintRun. - PrintRunData(big_o_data); - PrintRunData(rms_data); + for (const auto& run : reports) + PrintRunData(run); } void CSVReporter::PrintRunData(const Run & run) { @@ -125,29 +78,18 @@ void CSVReporter::PrintRunData(const Run & run) { return; } - double multiplier; - const char* timeLabel; - std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); - - double cpu_time = run.cpu_accumulated_time * multiplier; - double real_time = run.real_accumulated_time * multiplier; - if (run.iterations != 0) { - real_time = real_time / static_cast(run.iterations); - cpu_time = cpu_time / static_cast(run.iterations); - } - // Do not print iteration on bigO and RMS report if(!run.report_big_o && !run.report_rms) { Out << run.iterations; } Out << ","; - Out << real_time << ","; - Out << cpu_time << ","; + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; // Do not print timeLabel on RMS report if(!run.report_rms) { - Out << timeLabel; + Out << GetTimeUnitString(run.time_unit); } Out << ","; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index d3effe1..8ec18e0 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -98,71 +98,23 @@ void JSONReporter::ReportRuns(std::vector const& reports) { } first_report_ = false; - auto error_count = std::count_if( - reports.begin(), reports.end(), - [](Run const& run) {return run.error_occurred;}); - - std::vector reports_cp = reports; - if (reports.size() - error_count >= 2) { - Run mean_data; - Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); - reports_cp.push_back(mean_data); - reports_cp.push_back(stddev_data); - } - for (auto it = reports_cp.begin(); it != reports_cp.end(); ++it) { + for (auto it = reports.begin(); it != reports.end(); ++it) { out << indent << "{\n"; PrintRunData(*it); out << indent << '}'; auto it_cp = it; - if (++it_cp != reports_cp.end()) { + if (++it_cp != reports.end()) { out << ",\n"; } } } -void JSONReporter::ReportComplexity(const std::vector & complexity_reports) { - if (complexity_reports.size() < 2) { - // We don't report asymptotic complexity data if there was a single run. - return; - } - - std::string indent(4, ' '); - std::ostream& out = GetOutputStream(); - if (!first_report_) { - out << ",\n"; - } - - Run big_o_data; - Run rms_data; - BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data); - - // Output using PrintRun. - out << indent << "{\n"; - PrintRunData(big_o_data); - out << indent << "},\n"; - out << indent << "{\n"; - PrintRunData(rms_data); - out << indent << '}'; -} - void JSONReporter::Finalize() { // Close the list of benchmarks and the top level object. GetOutputStream() << "\n ]\n}\n"; } void JSONReporter::PrintRunData(Run const& run) { - double multiplier; - const char* timeLabel; - std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); - - double cpu_time = run.cpu_accumulated_time * multiplier; - double real_time = run.real_accumulated_time * multiplier; - if (run.iterations != 0) { - real_time = real_time / static_cast(run.iterations); - cpu_time = cpu_time / static_cast(run.iterations); - } - std::string indent(6, ' '); std::ostream& out = GetOutputStream(); out << indent @@ -182,13 +134,13 @@ void JSONReporter::PrintRunData(Run const& run) { << ",\n"; } out << indent - << FormatKV("real_time", RoundDouble(real_time)) + << FormatKV("real_time", RoundDouble(run.GetAdjustedRealTime())) << ",\n"; out << indent - << FormatKV("cpu_time", RoundDouble(cpu_time)); + << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime())); if(!run.report_rms) { out << ",\n" << indent - << FormatKV("time_unit", timeLabel); + << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); } if (run.bytes_per_second > 0.0) { out << ",\n" << indent diff --git a/src/reporter.cc b/src/reporter.cc index 5b303f7..5187859 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -13,7 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" -#include "complexity.h" +#include "walltime.h" #include @@ -31,131 +31,45 @@ BenchmarkReporter::BenchmarkReporter() { } -void BenchmarkReporter::ComputeStats( - const std::vector& reports, - Run* mean_data, Run* stddev_data) { - CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports"; - // Accumulators. - Stat1_d real_accumulated_time_stat; - Stat1_d cpu_accumulated_time_stat; - Stat1_d bytes_per_second_stat; - Stat1_d items_per_second_stat; - // All repetitions should be run with the same number of iterations so we - // can take this information from the first benchmark. - int64_t const run_iterations = reports.front().iterations; - - // Populate the accumulators. - for (Run const& run : reports) { - CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); - CHECK_EQ(run_iterations, run.iterations); - if (run.error_occurred) - continue; - real_accumulated_time_stat += - Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); - cpu_accumulated_time_stat += - Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); - items_per_second_stat += Stat1_d(run.items_per_second, run.iterations); - bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations); - } +BenchmarkReporter::~BenchmarkReporter() { +} - // Get the data from the accumulator to BenchmarkReporter::Run's. - mean_data->benchmark_name = reports[0].benchmark_name + "_mean"; - mean_data->iterations = run_iterations; - mean_data->real_accumulated_time = real_accumulated_time_stat.Mean() * - run_iterations; - mean_data->cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * - run_iterations; - mean_data->bytes_per_second = bytes_per_second_stat.Mean(); - mean_data->items_per_second = items_per_second_stat.Mean(); - - // Only add label to mean/stddev if it is same for all runs - mean_data->report_label = reports[0].report_label; - for (std::size_t i = 1; i < reports.size(); i++) { - if (reports[i].report_label != reports[0].report_label) { - mean_data->report_label = ""; - break; - } - } +void BenchmarkReporter::PrintBasicContext(std::ostream *out_ptr, + Context const &context) { + CHECK(out_ptr) << "cannot be null"; + auto& Out = *out_ptr; - stddev_data->benchmark_name = reports[0].benchmark_name + "_stddev"; - stddev_data->report_label = mean_data->report_label; - stddev_data->iterations = 0; - stddev_data->real_accumulated_time = - real_accumulated_time_stat.StdDev(); - stddev_data->cpu_accumulated_time = - cpu_accumulated_time_stat.StdDev(); - stddev_data->bytes_per_second = bytes_per_second_stat.StdDev(); - stddev_data->items_per_second = items_per_second_stat.StdDev(); -} + Out << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu + << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; -void BenchmarkReporter::ComputeBigO( - const std::vector& reports, - Run* big_o, Run* rms) { - CHECK(reports.size() >= 2) - << "Cannot compute asymptotic complexity for fewer than 2 reports"; - - // Accumulators. - std::vector n; - std::vector real_time; - std::vector cpu_time; - - // Populate the accumulators. - for (const Run& run : reports) { - n.push_back(run.complexity_n); - real_time.push_back(run.real_accumulated_time/run.iterations); - cpu_time.push_back(run.cpu_accumulated_time/run.iterations); + Out << LocalDateTimeString() << "\n"; + + if (context.cpu_scaling_enabled) { + Out << "***WARNING*** CPU scaling is enabled, the benchmark " + "real time measurements may be noisy and will incur extra " + "overhead.\n"; } - LeastSq result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - - // result_cpu.complexity is passed as parameter to result_real because in case - // reports[0].complexity is oAuto, the noise on the measured data could make - // the best fit function of Cpu and Real differ. In order to solve this, we - // take the best fitting function for the Cpu, and apply it to Real data. - LeastSq result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); - - std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); - - // Get the data from the accumulator to BenchmarkReporter::Run's. - big_o->benchmark_name = benchmark_name + "_BigO"; - big_o->iterations = 0; - big_o->real_accumulated_time = result_real.coef; - big_o->cpu_accumulated_time = result_cpu.coef; - big_o->report_big_o = true; - big_o->complexity = result_cpu.complexity; - - double multiplier; - const char* time_label; - std::tie(time_label, multiplier) = - GetTimeUnitAndMultiplier(reports[0].time_unit); - - // Only add label to mean/stddev if it is same for all runs - big_o->report_label = reports[0].report_label; - rms->benchmark_name = benchmark_name + "_RMS"; - rms->report_label = big_o->report_label; - rms->iterations = 0; - rms->real_accumulated_time = result_real.rms / multiplier; - rms->cpu_accumulated_time = result_cpu.rms / multiplier; - rms->report_rms = true; - rms->complexity = result_cpu.complexity; +#ifndef NDEBUG + Out << "***WARNING*** Library was built as DEBUG. Timings may be " + "affected.\n"; +#endif } -TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) { - switch (unit) { - case kMillisecond: - return std::make_pair("ms", 1e3); - case kMicrosecond: - return std::make_pair("us", 1e6); - case kNanosecond: - default: - return std::make_pair("ns", 1e9); - } +double BenchmarkReporter::Run::GetAdjustedRealTime() const { + double new_time = real_accumulated_time * GetTimeUnitMultiplier(time_unit); + if (iterations != 0) + new_time /= static_cast(iterations); + return new_time; } -void BenchmarkReporter::Finalize() { +double BenchmarkReporter::Run::GetAdjustedCPUTime() const { + double new_time = cpu_accumulated_time * GetTimeUnitMultiplier(time_unit); + if (iterations != 0) + new_time /= static_cast(iterations); + return new_time; } -BenchmarkReporter::~BenchmarkReporter() { -} + } // end namespace benchmark diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index acfbf68..2d2333b 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -84,11 +84,6 @@ public: rep->ReportRuns(report); } - virtual void ReportComplexity(const std::vector& complexity_reports) { - for (auto rep : reporters_) - rep->ReportComplexity(complexity_reports); - } - virtual void Finalize() { for (auto rep : reporters_) rep->Finalize(); -- cgit v1.2.3 From 02230445e065456371cea2a18d5fee6a33b0c1d9 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Fri, 27 May 2016 16:53:30 -0600 Subject: Move UnitTime helpers to reporter.h --- include/benchmark/benchmark_api.h | 24 ------------------------ include/benchmark/reporter.h | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 5929c66..e705d75 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -236,30 +236,6 @@ enum TimeUnit { kMillisecond }; -inline const char* GetTimeUnitString(TimeUnit unit) { - switch (unit) { - case kMillisecond: - return "ms"; - case kMicrosecond: - return "us"; - case kNanosecond: - default: - return "ns"; - } -} - -inline double GetTimeUnitMultiplier(TimeUnit unit) { - switch (unit) { - case kMillisecond: - return 1e3; - case kMicrosecond: - return 1e6; - case kNanosecond: - default: - return 1e9; - } -} - // BigO is passed to a benchmark in order to specify the asymptotic computational // complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index da53faa..0262754 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -187,5 +187,29 @@ private: void PrintRunData(const Run& report); }; +inline const char* GetTimeUnitString(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + default: + return "ns"; + } +} + +inline double GetTimeUnitMultiplier(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + default: + return 1e9; + } +} + } // end namespace benchmark #endif // BENCHMARK_REPORTER_H_ -- cgit v1.2.3 From 7188824c333d86fe999988ad61cbb0825780ab8b Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 30 May 2016 15:33:52 -0600 Subject: Fix windows build in console_reporter.cc. Fixes #228 --- src/console_reporter.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 87b3216..a10d4c6 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -38,8 +38,8 @@ bool ConsoleReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if (FLAGS_color_print && &Out != &GetOutputStream()) { - GetErrorString() << "Color printing is only supported for stdout on windows." + if (FLAGS_color_print && &std::cout != &GetOutputStream()) { + GetErrorStream() << "Color printing is only supported for stdout on windows." " Disabling color printing\n"; FLAGS_color_print = false; } -- cgit v1.2.3 From 519e8d0e56ac41813aa60bb6b39bf05308205b5b Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 30 May 2016 16:52:24 -0600 Subject: Fix missing declaration of FLAGS_color_print --- src/console_reporter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index a10d4c6..9b20ac8 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -30,6 +30,8 @@ #include "string_util.h" #include "walltime.h" +DECLARE_bool(color_print); + namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { -- cgit v1.2.3 From 74a278e206f9387dec72ec000436a18bfcb3070e Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 30 May 2016 17:13:41 -0600 Subject: Fix flaky test --- test/reporter_output_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 2d2333b..c09fbb6 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -146,8 +146,8 @@ ADD_CASES(&ConsoleOutputTests, { ADD_CASES(&JSONOutputTests, { {"\"name\": \"BM_basic\",$"}, {"\"iterations\": [0-9]+,$", MR_Next}, - {"\"real_time\": [0-9],$", MR_Next}, - {"\"cpu_time\": [0-9],$", MR_Next}, + {"\"real_time\": [0-9]{1,5},$", MR_Next}, + {"\"cpu_time\": [0-9]{1,5},$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next} }); -- cgit v1.2.3 From 3685cad68c5180e0da2afb7c05bd03422c0fe454 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Tue, 31 May 2016 14:52:26 -0700 Subject: Added new noexcept macros for VS2013 (#229) --- include/benchmark/macros.h | 10 +++++++++- src/check.h | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/benchmark/macros.h b/include/benchmark/macros.h index 3e9540e..09d13c1 100644 --- a/include/benchmark/macros.h +++ b/include/benchmark/macros.h @@ -28,15 +28,23 @@ # define BENCHMARK_UNUSED __attribute__((unused)) # define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) # define BENCHMARK_NOEXCEPT noexcept +# define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) #elif defined(_MSC_VER) && !defined(__clang__) # define BENCHMARK_UNUSED # define BENCHMARK_ALWAYS_INLINE __forceinline -# define BENCHMARK_NOEXCEPT +# if _MSC_VER >= 1900 +# define BENCHMARK_NOEXCEPT noexcept +# define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +# else +# define BENCHMARK_NOEXCEPT +# define BENCHMARK_NOEXCEPT_OP(x) +# endif # define __func__ __FUNCTION__ #else # define BENCHMARK_UNUSED # define BENCHMARK_ALWAYS_INLINE # define BENCHMARK_NOEXCEPT +# define BENCHMARK_NOEXCEPT_OP(x) #endif #if defined(__GNUC__) diff --git a/src/check.h b/src/check.h index cb49c49..4572bab 100644 --- a/src/check.h +++ b/src/check.h @@ -37,7 +37,7 @@ public: return log_; } - BENCHMARK_NORETURN ~CheckHandler() noexcept(false) { + BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) { log_ << std::endl; CallAbortHandler(); } -- cgit v1.2.3 From 84cd50b85e2c7ff384a362933446691d3c3be357 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Tue, 31 May 2016 16:54:02 -0700 Subject: Add workaround for VS2013 const-incorrect atomic (#230) --- src/benchmark.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index d86705e..599cda0 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -115,7 +115,13 @@ std::string* GetReportLabel() { // Global variable so that a benchmark can report an error as a human readable // string. If error_message is null no error occurred. -static std::atomic error_message = ATOMIC_VAR_INIT(nullptr); +#if defined(_MSC_VER) && _MSC_VER <= 1800 +typedef char* error_message_type; +#else +typedef const char* error_message_type; +#endif + +static std::atomic error_message = ATOMIC_VAR_INIT(nullptr); // TODO(ericwf): support MallocCounter. //static benchmark::MallocCounter *benchmark_mc; @@ -807,7 +813,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, MutexLock l(GetBenchmarkLock()); label = *GetReportLabel(); } - const char* error_msg = error_message; + error_message_type error_msg = error_message; const double min_time = !IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time; @@ -929,8 +935,9 @@ void State::ResumeTiming() { void State::SkipWithError(const char* msg) { CHECK(msg); error_occurred_ = true; - const char* expected_no_error_msg = nullptr; - error_message.compare_exchange_weak(expected_no_error_msg, msg); + error_message_type expected_no_error_msg = nullptr; + error_message.compare_exchange_weak(expected_no_error_msg, + const_cast(msg)); started_ = finished_ = true; total_iterations_ = max_iterations; timer_manager->RemoveErroredThread(); -- cgit v1.2.3 From 867f9145a0a45f8b993cec8b48309c19391acaa0 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 1 Jun 2016 23:08:01 +0200 Subject: added lambdas to complexity report --- include/benchmark/benchmark_api.h | 32 ++-- include/benchmark/reporter.h | 43 ++--- src/benchmark.cc | 167 ++++++++++--------- src/complexity.cc | 119 +++++++------- src/complexity.h | 20 +-- src/console_reporter.cc | 4 +- src/csv_reporter.cc | 7 +- src/json_reporter.cc | 29 +++- test/complexity_test.cc | 336 ++++++++++++++++++++++++++++++-------- 9 files changed, 494 insertions(+), 263 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index e705d75..8d5189a 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -152,6 +152,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include #include +#include #include "macros.h" @@ -247,15 +248,20 @@ enum BigO { oNCubed, oLogN, oNLogN, - oAuto + oAuto, + oLambda }; +// BigOFunc is passed to a benchmark in order to specify the asymptotic +// computational complexity for the benchmark. +typedef double(BigOFunc)(size_t); + // State is passed to a running Benchmark and contains state for the // benchmark to use. class State { public: State(size_t max_iters, bool has_x, int x, bool has_y, int y, - int thread_i, int n_threads); + int thread_i, int n_threads); // Returns true iff the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has @@ -268,13 +274,13 @@ public: } bool const res = total_iterations_++ < max_iterations; if (BENCHMARK_BUILTIN_EXPECT(!res, false)) { - assert(started_ && (!finished_ || error_occurred_)); - if (!error_occurred_) { - PauseTiming(); - } - // Total iterations now is one greater than max iterations. Fix this. - total_iterations_ = max_iterations; - finished_ = true; + assert(started_ && (!finished_ || error_occurred_)); + if (!error_occurred_) { + PauseTiming(); + } + // Total iterations now is one greater than max iterations. Fix this. + total_iterations_ = max_iterations; + finished_ = true; } return res; } @@ -359,7 +365,7 @@ public: // represent the length of N. BENCHMARK_ALWAYS_INLINE void SetComplexityN(size_t complexity_n) { - complexity_n_ = complexity_n; + complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE @@ -533,10 +539,14 @@ public: // to control how many iterations are run, and in the printing of items/second // or MB/second values. Benchmark* UseManualTime(); - + // Set the asymptotic computational complexity for the benchmark. If called // the asymptotic computational complexity will be shown on the output. Benchmark* Complexity(BigO complexity = benchmark::oAuto); + + // Set the asymptotic computational complexity for the benchmark. If called + // the asymptotic computational complexity will be shown on the output. + Benchmark* Complexity(BigOFunc* complexity); // Support for running multiple copies of the same benchmark concurrently // in multiple threads. This may be useful when measuring the scaling diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 0262754..4c7bff3 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -83,11 +83,12 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; - + // Keep track of arguments to compute asymptotic complexity - BigO complexity; - int complexity_n; - + BigO complexity; + BigOFunc* complexity_lambda; + size_t complexity_n; + // Inform print function whether the current run is a complexity report bool report_big_o; bool report_rms; @@ -113,7 +114,7 @@ class BenchmarkReporter { // 'reports' contains additional entries representing the asymptotic // complexity and RMS of that benchmark family. virtual void ReportRuns(const std::vector& report) = 0; - + // Called once and only once after ever group of benchmarks is run and // reported. virtual void Finalize() {} @@ -159,7 +160,7 @@ class ConsoleReporter : public BenchmarkReporter { virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); -protected: + protected: virtual void PrintRunData(const Run& report); size_t name_field_width_; @@ -189,25 +190,25 @@ private: inline const char* GetTimeUnitString(TimeUnit unit) { switch (unit) { - case kMillisecond: - return "ms"; - case kMicrosecond: - return "us"; - case kNanosecond: - default: - return "ns"; + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + default: + return "ns"; } } inline double GetTimeUnitMultiplier(TimeUnit unit) { - switch (unit) { - case kMillisecond: - return 1e3; - case kMicrosecond: - return 1e6; - case kNanosecond: - default: - return 1e9; + switch (unit) { + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + default: + return 1e9; } } diff --git a/src/benchmark.cc b/src/benchmark.cc index d86705e..2b55ac7 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -124,7 +124,7 @@ struct ThreadStats { ThreadStats() : bytes_processed(0), items_processed(0), complexity_n(0) {} int64_t bytes_processed; int64_t items_processed; - int complexity_n; + size_t complexity_n; }; // Timer management class @@ -140,7 +140,7 @@ class TimerManager { manual_time_used_(0), num_finalized_(0), phase_number_(0), - entered_(0) + entered_(0) { } @@ -277,11 +277,11 @@ class TimerManager { int phase_number_cp = phase_number_; auto cb = [this, phase_number_cp]() { return this->phase_number_ > phase_number_cp || - entered_ == running_threads_; // A thread has aborted in error + entered_ == running_threads_; // A thread has aborted in error }; phase_condition_.wait(ml.native_handle(), cb); if (phase_number_ > phase_number_cp) - return false; + return false; // else (running_threads_ == entered_) and we are the last thread. } // Last thread has reached the barrier @@ -311,6 +311,7 @@ struct Benchmark::Instance { bool use_real_time; bool use_manual_time; BigO complexity; + BigOFunc* complexity_lambda; bool last_benchmark_instance; int repetitions; double min_time; @@ -356,6 +357,7 @@ public: void UseRealTime(); void UseManualTime(); void Complexity(BigO complexity); + void ComplexityLambda(BigOFunc* complexity); void Threads(int t); void ThreadRange(int min_threads, int max_threads); void ThreadPerCpu(); @@ -376,6 +378,7 @@ private: bool use_real_time_; bool use_manual_time_; BigO complexity_; + BigOFunc* complexity_lambda_; std::vector thread_counts_; BenchmarkImp& operator=(BenchmarkImp const&); @@ -440,6 +443,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; instance.complexity = family->complexity_; + instance.complexity_lambda = family->complexity_lambda_; instance.threads = num_threads; instance.multithreaded = !(family->thread_counts_.empty()); @@ -567,6 +571,10 @@ void BenchmarkImp::Complexity(BigO complexity){ complexity_ = complexity; } +void BenchmarkImp::ComplexityLambda(BigOFunc* complexity) { + complexity_lambda_ = complexity; +} + void BenchmarkImp::Threads(int t) { CHECK_GT(t, 0); thread_counts_.push_back(t); @@ -691,6 +699,12 @@ Benchmark* Benchmark::Complexity(BigO complexity) { return this; } +Benchmark* Benchmark::Complexity(BigOFunc* complexity) { + imp_->Complexity(oLambda); + imp_->ComplexityLambda(complexity); + return this; +} + Benchmark* Benchmark::Threads(int t) { imp_->Threads(t); return this; @@ -717,7 +731,7 @@ void FunctionBenchmark::Run(State& st) { } // end namespace internal namespace { - + // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into *total. void RunInThread(const benchmark::internal::Benchmark::Instance* b, @@ -731,15 +745,15 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, MutexLock l(GetBenchmarkLock()); total->bytes_processed += st.bytes_processed(); total->items_processed += st.items_processed(); - total->complexity_n += st.complexity_length_n(); + total->complexity_n += st.complexity_length_n(); } timer_manager->Finalize(); } void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, - BenchmarkReporter* br, - std::vector& complexity_reports) + BenchmarkReporter* br, + std::vector& complexity_reports) EXCLUDES(GetBenchmarkLock()) { size_t iters = 1; @@ -750,7 +764,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, pool.resize(b.threads); const int repeats = b.repetitions != 0 ? b.repetitions - : FLAGS_benchmark_repetitions; + : FLAGS_benchmark_repetitions; for (int i = 0; i < repeats; i++) { std::string mem; for (;;) { @@ -830,27 +844,28 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.time_unit = b.time_unit; if (!report.error_occurred) { - double bytes_per_second = 0; - if (total.bytes_processed > 0 && seconds > 0.0) { - bytes_per_second = (total.bytes_processed / seconds); - } - double items_per_second = 0; - if (total.items_processed > 0 && seconds > 0.0) { - items_per_second = (total.items_processed / seconds); - } - - if (b.use_manual_time) { - report.real_accumulated_time = manual_accumulated_time; - } else { - report.real_accumulated_time = real_accumulated_time; - } - report.cpu_accumulated_time = cpu_accumulated_time; - report.bytes_per_second = bytes_per_second; - report.items_per_second = items_per_second; - report.complexity_n = total.complexity_n; - report.complexity = b.complexity; - if(report.complexity != oNone) - complexity_reports.push_back(report); + double bytes_per_second = 0; + if (total.bytes_processed > 0 && seconds > 0.0) { + bytes_per_second = (total.bytes_processed / seconds); + } + double items_per_second = 0; + if (total.items_processed > 0 && seconds > 0.0) { + items_per_second = (total.items_processed / seconds); + } + + if (b.use_manual_time) { + report.real_accumulated_time = manual_accumulated_time; + } else { + report.real_accumulated_time = real_accumulated_time; + } + report.cpu_accumulated_time = cpu_accumulated_time; + report.bytes_per_second = bytes_per_second; + report.items_per_second = items_per_second; + report.complexity_n = total.complexity_n; + report.complexity = b.complexity; + report.complexity_lambda = b.complexity_lambda; + if(report.complexity != oNone) + complexity_reports.push_back(report); } reports.push_back(report); @@ -878,17 +893,17 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, } std::vector additional_run_stats = ComputeStats(reports); reports.insert(reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); + additional_run_stats.end()); if((b.complexity != oNone) && b.last_benchmark_instance) { additional_run_stats = ComputeBigO(complexity_reports); reports.insert(reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); + additional_run_stats.end()); complexity_reports.clear(); } br->ReportRuns(reports); - + if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); @@ -949,56 +964,56 @@ void State::SetLabel(const char* label) { } namespace internal { -namespace { - -void RunMatchingBenchmarks(const std::vector& benchmarks, - BenchmarkReporter* reporter) { - CHECK(reporter != nullptr); - - // Determine the width of the name field using a minimum width of 10. - bool has_repetitions = FLAGS_benchmark_repetitions > 1; - size_t name_field_width = 10; - for (const Benchmark::Instance& benchmark : benchmarks) { - name_field_width = - std::max(name_field_width, benchmark.name.size()); - has_repetitions |= benchmark.repetitions > 1; - } - if (has_repetitions) - name_field_width += std::strlen("_stddev"); + namespace { + + void RunMatchingBenchmarks(const std::vector& benchmarks, + BenchmarkReporter* reporter) { + CHECK(reporter != nullptr); + + // Determine the width of the name field using a minimum width of 10. + bool has_repetitions = FLAGS_benchmark_repetitions > 1; + size_t name_field_width = 10; + for (const Benchmark::Instance& benchmark : benchmarks) { + name_field_width = + std::max(name_field_width, benchmark.name.size()); + has_repetitions |= benchmark.repetitions > 1; + } + if (has_repetitions) + name_field_width += std::strlen("_stddev"); - // Print header here - BenchmarkReporter::Context context; - context.num_cpus = NumCPUs(); - context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f; + // Print header here + BenchmarkReporter::Context context; + context.num_cpus = NumCPUs(); + context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f; - context.cpu_scaling_enabled = CpuScalingEnabled(); - context.name_field_width = name_field_width; + context.cpu_scaling_enabled = CpuScalingEnabled(); + context.name_field_width = name_field_width; - // Keep track of runing times of all instances of current benchmark - std::vector complexity_reports; + // Keep track of runing times of all instances of current benchmark + std::vector complexity_reports; - if (reporter->ReportContext(context)) { - for (const auto& benchmark : benchmarks) { - RunBenchmark(benchmark, reporter, complexity_reports); + if (reporter->ReportContext(context)) { + for (const auto& benchmark : benchmarks) { + RunBenchmark(benchmark, reporter, complexity_reports); + } + } } - } -} -std::unique_ptr GetDefaultReporter() { - typedef std::unique_ptr PtrType; - if (FLAGS_benchmark_format == "console") { - return PtrType(new ConsoleReporter); - } else if (FLAGS_benchmark_format == "json") { - return PtrType(new JSONReporter); - } else if (FLAGS_benchmark_format == "csv") { - return PtrType(new CSVReporter); - } else { - std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n"; - std::exit(1); - } -} + std::unique_ptr GetDefaultReporter() { + typedef std::unique_ptr PtrType; + if (FLAGS_benchmark_format == "console") { + return PtrType(new ConsoleReporter); + } else if (FLAGS_benchmark_format == "json") { + return PtrType(new JSONReporter); + } else if (FLAGS_benchmark_format == "csv") { + return PtrType(new CSVReporter); + } else { + std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n"; + std::exit(1); + } + } -} // end namespace + } // end namespace } // end namespace internal size_t RunSpecifiedBenchmarks() { diff --git a/src/complexity.cc b/src/complexity.cc index 3e42f5d..97f86d8 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -25,43 +25,43 @@ #include namespace benchmark { - + // Internal function to calculate the different scalability forms -std::function FittingCurve(BigO complexity) { +BigOFunc* FittingCurve(BigO complexity) { switch (complexity) { - case oN: - return [](int n) {return n; }; - case oNSquared: - return [](int n) {return n*n; }; - case oNCubed: - return [](int n) {return n*n*n; }; - case oLogN: - return [](int n) {return log2(n); }; - case oNLogN: - return [](int n) {return n * log2(n); }; - case o1: - default: - return [](int) {return 1; }; + case oN: + return [](size_t n) -> double {return n; }; + case oNSquared: + return [](size_t n) -> double {return n * n; }; + case oNCubed: + return [](size_t n) -> double {return n * n * n; }; + case oLogN: + return [](size_t n) {return log2(n); }; + case oNLogN: + return [](size_t n) {return n * log2(n); }; + case o1: + default: + return [](size_t) {return 1.0; }; } } // Function to return an string for the calculated complexity std::string GetBigOString(BigO complexity) { switch (complexity) { - case oN: - return "* N"; - case oNSquared: - return "* N**2"; - case oNCubed: - return "* N**3"; - case oLogN: - return "* lgN"; - case oNLogN: - return "* NlgN"; - case o1: - return "* 1"; - default: - return ""; + case oN: + return "N"; + case oNSquared: + return "N^2"; + case oNCubed: + return "N^3"; + case oLogN: + return "lgN"; + case oNLogN: + return "NlgN"; + case o1: + return "(1)"; + default: + return "f(N)"; } } @@ -75,21 +75,9 @@ std::string GetBigOString(BigO complexity) { // For a deeper explanation on the algorithm logic, look the README file at // http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -// This interface is currently not used from the oustide, but it has been -// provided for future upgrades. If in the future it is not needed to support -// Cxx03, then all the calculations could be upgraded to use lambdas because -// they are more powerful and provide a cleaner inferface than enumerators, -// but complete implementation with lambdas will not work for Cxx03 -// (e.g. lack of std::function). -// In case lambdas are implemented, the interface would be like : -// -> Complexity([](int n) {return n;};) -// and any arbitrary and valid equation would be allowed, but the option to -// calculate the best fit to the most common scalability curves will still -// be kept. - -LeastSq CalculateLeastSq(const std::vector& n, - const std::vector& time, - std::function fitting_curve) { +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + BigOFunc* fitting_curve) { double sigma_gn = 0.0; double sigma_gn_squared = 0.0; double sigma_time = 0.0; @@ -105,6 +93,7 @@ LeastSq CalculateLeastSq(const std::vector& n, } LeastSq result; + result.complexity = oLambda; // Calculate complexity. result.coef = sigma_time_gn / sigma_gn_squared; @@ -144,19 +133,19 @@ LeastSq MinimalLeastSq(const std::vector& n, oLogN, oN, oNLogN, oNSquared, oNCubed }; // Take o1 as default best fitting curve - best_fit = CalculateLeastSq(n, time, FittingCurve(o1)); + best_fit = MinimalLeastSq(n, time, FittingCurve(o1)); best_fit.complexity = o1; // Compute all possible fitting curves and stick to the best one for (const auto& fit : fit_curves) { - LeastSq current_fit = CalculateLeastSq(n, time, FittingCurve(fit)); + LeastSq current_fit = MinimalLeastSq(n, time, FittingCurve(fit)); if (current_fit.rms < best_fit.rms) { best_fit = current_fit; best_fit.complexity = fit; } } } else { - best_fit = CalculateLeastSq(n, time, FittingCurve(complexity)); + best_fit = MinimalLeastSq(n, time, FittingCurve(complexity)); best_fit.complexity = complexity; } @@ -164,14 +153,14 @@ LeastSq MinimalLeastSq(const std::vector& n, } std::vector ComputeStats( - const std::vector& reports) + const std::vector& reports) { typedef BenchmarkReporter::Run Run; std::vector results; auto error_count = std::count_if( - reports.begin(), reports.end(), - [](Run const& run) {return run.error_occurred;}); + reports.begin(), reports.end(), + [](Run const& run) {return run.error_occurred;}); if (reports.size() - error_count < 2) { // We don't report aggregated data if there was a single run. @@ -193,9 +182,9 @@ std::vector ComputeStats( if (run.error_occurred) continue; real_accumulated_time_stat += - Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); + Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); cpu_accumulated_time_stat += - Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); + Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); items_per_second_stat += Stat1_d(run.items_per_second, run.iterations); bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations); } @@ -205,9 +194,9 @@ std::vector ComputeStats( mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; mean_data.iterations = run_iterations; mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * - run_iterations; + run_iterations; mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * - run_iterations; + run_iterations; mean_data.bytes_per_second = bytes_per_second_stat.Mean(); mean_data.items_per_second = items_per_second_stat.Mean(); @@ -225,9 +214,9 @@ std::vector ComputeStats( stddev_data.report_label = mean_data.report_label; stddev_data.iterations = 0; stddev_data.real_accumulated_time = - real_accumulated_time_stat.StdDev(); + real_accumulated_time_stat.StdDev(); stddev_data.cpu_accumulated_time = - cpu_accumulated_time_stat.StdDev(); + cpu_accumulated_time_stat.StdDev(); stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); stddev_data.items_per_second = items_per_second_stat.StdDev(); @@ -237,7 +226,7 @@ std::vector ComputeStats( } std::vector ComputeBigO( - const std::vector& reports) + const std::vector& reports) { typedef BenchmarkReporter::Run Run; std::vector results; @@ -256,14 +245,16 @@ std::vector ComputeBigO( cpu_time.push_back(run.cpu_accumulated_time/run.iterations); } - LeastSq result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - - // result_cpu.complexity is passed as parameter to result_real because in case - // reports[0].complexity is oAuto, the noise on the measured data could make - // the best fit function of Cpu and Real differ. In order to solve this, we - // take the best fitting function for the Cpu, and apply it to Real data. - LeastSq result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + LeastSq result_cpu; + LeastSq result_real; + if (reports[0].complexity != oLambda) { + result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); + result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + } else { + result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); + result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); + } std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. diff --git a/src/complexity.h b/src/complexity.h index be095a9..798154a 100644 --- a/src/complexity.h +++ b/src/complexity.h @@ -26,15 +26,15 @@ namespace benchmark { -// Return a vector containing the mean and standard devation information for -// the specified list of reports. If 'reports' contains less than two -// non-errored runs an empty vector is returned -std::vector ComputeStats( + // Return a vector containing the mean and standard devation information for + // the specified list of reports. If 'reports' contains less than two + // non-errored runs an empty vector is returned + std::vector ComputeStats( const std::vector& reports); -// Return a vector containing the bigO and RMS information for the specified -// list of reports. If 'reports.size() < 2' an empty vector is returned. -std::vector ComputeBigO( + // Return a vector containing the bigO and RMS information for the specified + // list of reports. If 'reports.size() < 2' an empty vector is returned. + std::vector ComputeBigO( const std::vector& reports); // This data structure will contain the result returned by MinimalLeastSq @@ -60,11 +60,5 @@ struct LeastSq { // Function to return an string for the calculated complexity std::string GetBigOString(BigO complexity); -// Find the coefficient for the high-order term in the running time, by -// minimizing the sum of squares of relative error. -LeastSq MinimalLeastSq(const std::vector& n, - const std::vector& time, - const BigO complexity = oAuto); - } // end namespace benchmark #endif // COMPLEXITY_H_ diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 9b20ac8..2783097 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -90,8 +90,8 @@ void ConsoleReporter::PrintRunData(const Run& result) { const double cpu_time = result.GetAdjustedCPUTime(); if(result.report_big_o) { - std::string big_o = result.report_big_o ? GetBigOString(result.complexity) : ""; - ColorPrintf(Out, COLOR_YELLOW, "%10.4f %s %10.4f %s ", + std::string big_o = GetBigOString(result.complexity); + ColorPrintf(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", real_time, big_o.c_str(), cpu_time, big_o.c_str()); } else if(result.report_rms) { ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 5c18c9e..775a46c 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" +#include "complexity.h" #include #include @@ -87,8 +88,10 @@ void CSVReporter::PrintRunData(const Run & run) { Out << run.GetAdjustedRealTime() << ","; Out << run.GetAdjustedCPUTime() << ","; - // Do not print timeLabel on RMS report - if(!run.report_rms) { + // Do not print timeLabel on bigO and RMS report + if(run.report_big_o) { + Out << GetBigOString(run.complexity); + } else if(!run.report_rms){ Out << GetTimeUnitString(run.time_unit); } Out << ","; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 8ec18e0..04cb490 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/reporter.h" +#include "complexity.h" #include #include @@ -132,15 +133,29 @@ void JSONReporter::PrintRunData(Run const& run) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; - } - out << indent - << FormatKV("real_time", RoundDouble(run.GetAdjustedRealTime())) - << ",\n"; - out << indent - << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime())); - if(!run.report_rms) { + out << indent + << FormatKV("real_time", RoundDouble(run.GetAdjustedRealTime())) + << ",\n"; + out << indent + << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime())); out << ",\n" << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); + } else if(run.report_big_o) { + out << indent + << FormatKV("cpu_coefficient", RoundDouble(run.GetAdjustedCPUTime())) + << ",\n"; + out << indent + << FormatKV("real_coefficient", RoundDouble(run.GetAdjustedRealTime())) + << ",\n"; + out << indent + << FormatKV("big_o", GetBigOString(run.complexity)) + << ",\n"; + out << indent + << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); + } else if(run.report_rms) { + out << indent + << FormatKV("rms", RoundDouble(run.GetAdjustedCPUTime()*100)) + << "%"; } if (run.bytes_per_second > 0.0) { out << ",\n" << indent diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 225a181..9ef1f10 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -1,37 +1,193 @@ -#include "benchmark/benchmark_api.h" - -#include -#include +#undef NDEBUG +#include "benchmark/benchmark.h" +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "../src/re.h" // NOTE: re.h is for internal use only +#include +#include +#include +#include #include -#include +#include #include -std::vector ConstructRandomVector(int size) { - std::vector v; - v.reserve(size); - for (int i = 0; i < size; ++i) { - v.push_back(rand() % size); +namespace { + +// ========================================================================= // +// -------------------------- Testing Case --------------------------------- // +// ========================================================================= // + +enum MatchRules { + MR_Default, // Skip non-matching lines until a match is found. + MR_Next // Match must occur on the next line. +}; + +struct TestCase { + std::string regex; + int match_rule; + + TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {} + + void Check(std::stringstream& remaining_output) const { + benchmark::Regex r; + std::string err_str; + r.Init(regex, &err_str); + CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\"" + << " got Error: " << err_str; + + std::string line; + while (remaining_output.eof() == false) { + CHECK(remaining_output.good()); + std::getline(remaining_output, line); + if (r.Match(line)) return; + CHECK(match_rule != MR_Next) << "Expected line \"" << line + << "\" to match regex \"" << regex << "\""; + } + + CHECK(remaining_output.eof() == false) + << "End of output reached before match for regex \"" << regex + << "\" was found"; } - return v; -} +}; -std::map ConstructRandomMap(int size) { - std::map m; - for (int i = 0; i < size; ++i) { - m.insert(std::make_pair(rand() % size, rand() % size)); +std::vector ConsoleOutputTests; +std::vector JSONOutputTests; +std::vector CSVOutputTests; + +// ========================================================================= // +// -------------------------- Test Helpers --------------------------------- // +// ========================================================================= // + +class TestReporter : public benchmark::BenchmarkReporter { +public: + TestReporter(std::vector reps) + : reporters_(reps) {} + + virtual bool ReportContext(const Context& context) { + bool last_ret = false; + bool first = true; + for (auto rep : reporters_) { + bool new_ret = rep->ReportContext(context); + CHECK(first || new_ret == last_ret) + << "Reports return different values for ReportContext"; + first = false; + last_ret = new_ret; + } + return last_ret; } - return m; + + virtual void ReportRuns(const std::vector& report) { + for (auto rep : reporters_) + rep->ReportRuns(report); + } + + virtual void Finalize() { + for (auto rep : reporters_) + rep->Finalize(); + } + +private: + std::vector reporters_; +}; + + +#define CONCAT2(x, y) x##y +#define CONCAT(x, y) CONCAT2(x, y) + +#define ADD_CASES(...) \ + int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +int AddCases(std::vector* out, std::initializer_list const& v) { + for (auto const& TC : v) + out->push_back(TC); + return 0; +} + +template +std::string join(First f) { return f; } + +template +std::string join(First f, Args&&... args) { + return std::string(std::move(f)) + "[ ]+" + join(std::forward(args)...); } +std::string dec_re = "[0-9]+\\.[0-9]+"; + +#define ADD_COMPLEXITY_CASES(...) \ + int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) + +int AddComplexityTest(std::vector* console_out, std::vector* json_out, + std::vector* csv_out, std::string big_o_test_name, + std::string rms_test_name, std::string big_o) { + std::string big_o_str = dec_re + " " + big_o; + AddCases(console_out, { + {join("^" + big_o_test_name + "", big_o_str, big_o_str) + "[ ]*$"}, + {join("^" + rms_test_name + "", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"} + }); + AddCases(json_out, { + {"\"name\": \"" + big_o_test_name + "\",$"}, + {"\"cpu_coefficient\": [0-9]+,$", MR_Next}, + {"\"real_coefficient\": [0-9]{1,5},$", MR_Next}, + {"\"big_o\": \"" + big_o + "\",$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}, + {"\"name\": \"" + rms_test_name + "\",$"}, + {"\"rms\": [0-9]+%$", MR_Next}, + {"}", MR_Next} + }); + AddCases(csv_out, { + {"^\"" + big_o_test_name + "\",," + dec_re + "," + dec_re + "," + big_o + ",,,,,$"}, + {"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$"} + }); + return 0; +} + +} // end namespace + +// ========================================================================= // +// --------------------------- Testing BigO O(1) --------------------------- // +// ========================================================================= // + void BM_Complexity_O1(benchmark::State& state) { while (state.KeepRunning()) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(); +BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity([](size_t){return 1.0; }); + +std::string big_o_1_test_name = "BM_Complexity_O1_BigO"; +std::string rms_o_1_test_name = "BM_Complexity_O1_RMS"; +std::string enum_auto_big_o_1 = "\\([0-9]+\\)"; +std::string lambda_big_o_1 = "f\\(N\\)"; + +// Add automatic tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); + +// Add enum tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); + +// Add lambda tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); + +// ========================================================================= // +// --------------------------- Testing BigO O(N) --------------------------- // +// ========================================================================= // + +std::vector ConstructRandomVector(int size) { + std::vector v; + v.reserve(size); + for (int i = 0; i < size; ++i) { + v.push_back(rand() % size); + } + return v; +} -static void BM_Complexity_O_N(benchmark::State& state) { +void BM_Complexity_O_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector) while (state.KeepRunning()) { @@ -39,51 +195,30 @@ static void BM_Complexity_O_N(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) -> double{return n; }); -static void BM_Complexity_O_N_Squared(benchmark::State& state) { - std::string s1(state.range_x(), '-'); - std::string s2(state.range_x(), '-'); - state.SetComplexityN(state.range_x()); - while (state.KeepRunning()) - for(char& c1 : s1) { - for(char& c2 : s2) { - benchmark::DoNotOptimize(c1 = 'a'); - benchmark::DoNotOptimize(c2 = 'b'); - } - } -} -BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::oNSquared); +std::string big_o_n_test_name = "BM_Complexity_O_N_BigO"; +std::string rms_o_n_test_name = "BM_Complexity_O_N_RMS"; +std::string enum_auto_big_o_n = "N"; +std::string lambda_big_o_n = "f\\(N\\)"; -static void BM_Complexity_O_N_Cubed(benchmark::State& state) { - std::string s1(state.range_x(), '-'); - std::string s2(state.range_x(), '-'); - std::string s3(state.range_x(), '-'); - state.SetComplexityN(state.range_x()); - while (state.KeepRunning()) - for(char& c1 : s1) { - for(char& c2 : s2) { - for(char& c3 : s3) { - benchmark::DoNotOptimize(c1 = 'a'); - benchmark::DoNotOptimize(c2 = 'b'); - benchmark::DoNotOptimize(c3 = 'c'); - } - } - } -} -BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::oNCubed); +// Add automatic tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); -static void BM_Complexity_O_log_N(benchmark::State& state) { - auto m = ConstructRandomMap(state.range_x()); - const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector) - while (state.KeepRunning()) { - benchmark::DoNotOptimize(m.find(item_not_in_vector)); - } - state.SetComplexityN(state.range_x()); -} -BENCHMARK(BM_Complexity_O_log_N) - -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oLogN); +// Add enum tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); + +// Add lambda tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); + +// ========================================================================= // +// ------------------------- Testing BigO O(N*lgN) ------------------------- // +// ========================================================================= // static void BM_Complexity_O_N_log_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range_x()); @@ -92,15 +227,82 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) {return n * log2(n); }); -// Test benchmark with no range and check no complexity is calculated. -void BM_Extreme_Cases(benchmark::State& state) { - while (state.KeepRunning()) { +std::string big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; +std::string rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; +std::string enum_auto_big_o_n_lg_n = "NlgN"; +std::string lambda_big_o_n_lg_n = "f\\(N\\)"; + +// Add automatic tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); + +// Add enum tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); + +// Add lambda tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); + + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + + +int main(int argc, char* argv[]) { + // Add --color_print=false to argv since we don't want to match color codes. + char new_arg[64]; + char* new_argv[64]; + std::copy(argv, argv + argc, new_argv); + new_argv[argc++] = std::strcpy(new_arg, "--color_print=false"); + benchmark::Initialize(&argc, new_argv); + + benchmark::ConsoleReporter CR; + benchmark::JSONReporter JR; + benchmark::CSVReporter CSVR; + struct ReporterTest { + const char* name; + std::vector& output_cases; + benchmark::BenchmarkReporter& reporter; + std::stringstream out_stream; + std::stringstream err_stream; + + ReporterTest(const char* n, + std::vector& out_tc, + benchmark::BenchmarkReporter& br) + : name(n), output_cases(out_tc), reporter(br) { + reporter.SetOutputStream(&out_stream); + reporter.SetErrorStream(&err_stream); + } + } TestCases[] = { + {"ConsoleReporter", ConsoleOutputTests, CR}, + {"JSONReporter", JSONOutputTests, JR}, + {"CSVReporter", CSVOutputTests, CSVR} + }; + + // Create the test reporter and run the benchmarks. + std::cout << "Running benchmarks...\n"; + TestReporter test_rep({&CR, &JR, &CSVR}); + benchmark::RunSpecifiedBenchmarks(&test_rep); + + for (auto& rep_test : TestCases) { + std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string banner(msg.size() - 1, '-'); + std::cout << banner << msg << banner << "\n"; + + std::cerr << rep_test.err_stream.str(); + std::cout << rep_test.out_stream.str(); + + for (const auto& TC : rep_test.output_cases) + TC.Check(rep_test.out_stream); + + std::cout << "\n"; } + return 0; } -BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::oNLogN); -BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(); -BENCHMARK_MAIN() -- cgit v1.2.3 From 8c73d49b775610a4ee027a7f44b37962684a2370 Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 1 Jun 2016 23:13:10 +0200 Subject: fixed reporter_output_test --- test/reporter_output_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index c09fbb6..b3898ac 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -189,7 +189,7 @@ void BM_Complexity_O1(benchmark::State& state) { } BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); -std::string bigOStr = "[0-9]+\\.[0-9]+ \\* [0-9]+"; +std::string bigOStr = "[0-9]+\\.[0-9]+ \\([0-9]+\\)"; ADD_CASES(&ConsoleOutputTests, { {join("^BM_Complexity_O1_BigO", bigOStr, bigOStr) + "[ ]*$"}, -- cgit v1.2.3 From 212cfe1c2e659d9523cbc453917d0cdde4699bcd Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 19:01:10 +0200 Subject: removed check on automatic fit, to avoid random convergence misfits breaking the build --- test/complexity_test.cc | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 9ef1f10..ed434fe 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -153,19 +153,15 @@ void BM_Complexity_O1(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(); BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity([](size_t){return 1.0; }); +BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(); std::string big_o_1_test_name = "BM_Complexity_O1_BigO"; std::string rms_o_1_test_name = "BM_Complexity_O1_RMS"; std::string enum_auto_big_o_1 = "\\([0-9]+\\)"; std::string lambda_big_o_1 = "f\\(N\\)"; -// Add automatic tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); - // Add enum tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); @@ -195,19 +191,15 @@ void BM_Complexity_O_N(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) -> double{return n; }); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); std::string big_o_n_test_name = "BM_Complexity_O_N_BigO"; std::string rms_o_n_test_name = "BM_Complexity_O_N_RMS"; std::string enum_auto_big_o_n = "N"; std::string lambda_big_o_n = "f\\(N\\)"; -// Add automatic tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); - // Add enum tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); @@ -227,19 +219,15 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) {return n * log2(n); }); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); std::string big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; std::string rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; std::string enum_auto_big_o_n_lg_n = "NlgN"; std::string lambda_big_o_n_lg_n = "f\\(N\\)"; -// Add automatic tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); - // Add enum tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); -- cgit v1.2.3 From 11e304355492670709c60e6d39eb42fc01fd878a Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 19:42:08 +0200 Subject: checked format before pull request --- include/benchmark/benchmark_api.h | 16 ++-- include/benchmark/reporter.h | 10 +-- src/benchmark.cc | 150 +++++++++++++++++++------------------- src/complexity.cc | 4 +- src/complexity.h | 14 ++-- src/json_reporter.cc | 2 +- 6 files changed, 98 insertions(+), 98 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 8d5189a..49167ff 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -261,16 +261,16 @@ typedef double(BigOFunc)(size_t); class State { public: State(size_t max_iters, bool has_x, int x, bool has_y, int y, - int thread_i, int n_threads); + int thread_i, int n_threads); - // Returns true iff the benchmark should continue through another iteration. + // Returns true if the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has // returned false. bool KeepRunning() { if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { - assert(!finished_); - started_ = true; - ResumeTiming(); + assert(!finished_); + started_ = true; + ResumeTiming(); } bool const res = total_iterations_++ < max_iterations; if (BENCHMARK_BUILTIN_EXPECT(!res, false)) { @@ -365,7 +365,7 @@ public: // represent the length of N. BENCHMARK_ALWAYS_INLINE void SetComplexityN(size_t complexity_n) { - complexity_n_ = complexity_n; + complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE @@ -539,11 +539,11 @@ public: // to control how many iterations are run, and in the printing of items/second // or MB/second values. Benchmark* UseManualTime(); - + // Set the asymptotic computational complexity for the benchmark. If called // the asymptotic computational complexity will be shown on the output. Benchmark* Complexity(BigO complexity = benchmark::oAuto); - + // Set the asymptotic computational complexity for the benchmark. If called // the asymptotic computational complexity will be shown on the output. Benchmark* Complexity(BigOFunc* complexity); diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 4c7bff3..f37e0a3 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -83,12 +83,12 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; - + // Keep track of arguments to compute asymptotic complexity BigO complexity; BigOFunc* complexity_lambda; size_t complexity_n; - + // Inform print function whether the current run is a complexity report bool report_big_o; bool report_rms; @@ -114,7 +114,7 @@ class BenchmarkReporter { // 'reports' contains additional entries representing the asymptotic // complexity and RMS of that benchmark family. virtual void ReportRuns(const std::vector& report) = 0; - + // Called once and only once after ever group of benchmarks is run and // reported. virtual void Finalize() {} @@ -156,11 +156,11 @@ private: // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). class ConsoleReporter : public BenchmarkReporter { - public: +public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); - protected: +protected: virtual void PrintRunData(const Run& report); size_t name_field_width_; diff --git a/src/benchmark.cc b/src/benchmark.cc index 2b55ac7..c56faa9 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -140,7 +140,7 @@ class TimerManager { manual_time_used_(0), num_finalized_(0), phase_number_(0), - entered_(0) + entered_(0) { } @@ -277,7 +277,7 @@ class TimerManager { int phase_number_cp = phase_number_; auto cb = [this, phase_number_cp]() { return this->phase_number_ > phase_number_cp || - entered_ == running_threads_; // A thread has aborted in error + entered_ == running_threads_; // A thread has aborted in error }; phase_condition_.wait(ml.native_handle(), cb); if (phase_number_ > phase_number_cp) @@ -731,7 +731,7 @@ void FunctionBenchmark::Run(State& st) { } // end namespace internal namespace { - + // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into *total. void RunInThread(const benchmark::internal::Benchmark::Instance* b, @@ -745,15 +745,15 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, MutexLock l(GetBenchmarkLock()); total->bytes_processed += st.bytes_processed(); total->items_processed += st.items_processed(); - total->complexity_n += st.complexity_length_n(); + total->complexity_n += st.complexity_length_n(); } timer_manager->Finalize(); } void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, - BenchmarkReporter* br, - std::vector& complexity_reports) + BenchmarkReporter* br, + std::vector& complexity_reports) EXCLUDES(GetBenchmarkLock()) { size_t iters = 1; @@ -764,7 +764,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, pool.resize(b.threads); const int repeats = b.repetitions != 0 ? b.repetitions - : FLAGS_benchmark_repetitions; + : FLAGS_benchmark_repetitions; for (int i = 0; i < repeats; i++) { std::string mem; for (;;) { @@ -844,28 +844,28 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.time_unit = b.time_unit; if (!report.error_occurred) { - double bytes_per_second = 0; - if (total.bytes_processed > 0 && seconds > 0.0) { - bytes_per_second = (total.bytes_processed / seconds); - } - double items_per_second = 0; - if (total.items_processed > 0 && seconds > 0.0) { - items_per_second = (total.items_processed / seconds); - } - - if (b.use_manual_time) { - report.real_accumulated_time = manual_accumulated_time; - } else { - report.real_accumulated_time = real_accumulated_time; - } - report.cpu_accumulated_time = cpu_accumulated_time; - report.bytes_per_second = bytes_per_second; - report.items_per_second = items_per_second; - report.complexity_n = total.complexity_n; - report.complexity = b.complexity; - report.complexity_lambda = b.complexity_lambda; - if(report.complexity != oNone) - complexity_reports.push_back(report); + double bytes_per_second = 0; + if (total.bytes_processed > 0 && seconds > 0.0) { + bytes_per_second = (total.bytes_processed / seconds); + } + double items_per_second = 0; + if (total.items_processed > 0 && seconds > 0.0) { + items_per_second = (total.items_processed / seconds); + } + + if (b.use_manual_time) { + report.real_accumulated_time = manual_accumulated_time; + } else { + report.real_accumulated_time = real_accumulated_time; + } + report.cpu_accumulated_time = cpu_accumulated_time; + report.bytes_per_second = bytes_per_second; + report.items_per_second = items_per_second; + report.complexity_n = total.complexity_n; + report.complexity = b.complexity; + report.complexity_lambda = b.complexity_lambda; + if(report.complexity != oNone) + complexity_reports.push_back(report); } reports.push_back(report); @@ -893,17 +893,17 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, } std::vector additional_run_stats = ComputeStats(reports); reports.insert(reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); + additional_run_stats.end()); if((b.complexity != oNone) && b.last_benchmark_instance) { additional_run_stats = ComputeBigO(complexity_reports); reports.insert(reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); + additional_run_stats.end()); complexity_reports.clear(); } br->ReportRuns(reports); - + if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); @@ -964,56 +964,56 @@ void State::SetLabel(const char* label) { } namespace internal { - namespace { - - void RunMatchingBenchmarks(const std::vector& benchmarks, - BenchmarkReporter* reporter) { - CHECK(reporter != nullptr); - - // Determine the width of the name field using a minimum width of 10. - bool has_repetitions = FLAGS_benchmark_repetitions > 1; - size_t name_field_width = 10; - for (const Benchmark::Instance& benchmark : benchmarks) { - name_field_width = - std::max(name_field_width, benchmark.name.size()); - has_repetitions |= benchmark.repetitions > 1; - } - if (has_repetitions) - name_field_width += std::strlen("_stddev"); +namespace { - // Print header here - BenchmarkReporter::Context context; - context.num_cpus = NumCPUs(); - context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f; +void RunMatchingBenchmarks(const std::vector& benchmarks, + BenchmarkReporter* reporter) { + CHECK(reporter != nullptr); + + // Determine the width of the name field using a minimum width of 10. + bool has_repetitions = FLAGS_benchmark_repetitions > 1; + size_t name_field_width = 10; + for (const Benchmark::Instance& benchmark : benchmarks) { + name_field_width = + std::max(name_field_width, benchmark.name.size()); + has_repetitions |= benchmark.repetitions > 1; + } + if (has_repetitions) + name_field_width += std::strlen("_stddev"); - context.cpu_scaling_enabled = CpuScalingEnabled(); - context.name_field_width = name_field_width; + // Print header here + BenchmarkReporter::Context context; + context.num_cpus = NumCPUs(); + context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f; - // Keep track of runing times of all instances of current benchmark - std::vector complexity_reports; + context.cpu_scaling_enabled = CpuScalingEnabled(); + context.name_field_width = name_field_width; - if (reporter->ReportContext(context)) { - for (const auto& benchmark : benchmarks) { - RunBenchmark(benchmark, reporter, complexity_reports); - } - } - } + // Keep track of runing times of all instances of current benchmark + std::vector complexity_reports; - std::unique_ptr GetDefaultReporter() { - typedef std::unique_ptr PtrType; - if (FLAGS_benchmark_format == "console") { - return PtrType(new ConsoleReporter); - } else if (FLAGS_benchmark_format == "json") { - return PtrType(new JSONReporter); - } else if (FLAGS_benchmark_format == "csv") { - return PtrType(new CSVReporter); - } else { - std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n"; - std::exit(1); - } + if (reporter->ReportContext(context)) { + for (const auto& benchmark : benchmarks) { + RunBenchmark(benchmark, reporter, complexity_reports); } + } +} - } // end namespace +std::unique_ptr GetDefaultReporter() { + typedef std::unique_ptr PtrType; + if (FLAGS_benchmark_format == "console") { + return PtrType(new ConsoleReporter); + } else if (FLAGS_benchmark_format == "json") { + return PtrType(new JSONReporter); + } else if (FLAGS_benchmark_format == "csv") { + return PtrType(new CSVReporter); + } else { + std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n"; + std::exit(1); + } +} + +} // end namespace } // end namespace internal size_t RunSpecifiedBenchmarks() { diff --git a/src/complexity.cc b/src/complexity.cc index 97f86d8..0d6f90b 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -194,9 +194,9 @@ std::vector ComputeStats( mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; mean_data.iterations = run_iterations; mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * - run_iterations; + run_iterations; mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * - run_iterations; + run_iterations; mean_data.bytes_per_second = bytes_per_second_stat.Mean(); mean_data.items_per_second = items_per_second_stat.Mean(); diff --git a/src/complexity.h b/src/complexity.h index 798154a..85cc125 100644 --- a/src/complexity.h +++ b/src/complexity.h @@ -26,15 +26,15 @@ namespace benchmark { - // Return a vector containing the mean and standard devation information for - // the specified list of reports. If 'reports' contains less than two - // non-errored runs an empty vector is returned - std::vector ComputeStats( +// Return a vector containing the mean and standard devation information for +// the specified list of reports. If 'reports' contains less than two +// non-errored runs an empty vector is returned +std::vector ComputeStats( const std::vector& reports); - // Return a vector containing the bigO and RMS information for the specified - // list of reports. If 'reports.size() < 2' an empty vector is returned. - std::vector ComputeBigO( +// Return a vector containing the bigO and RMS information for the specified +// list of reports. If 'reports.size() < 2' an empty vector is returned. +std::vector ComputeBigO( const std::vector& reports); // This data structure will contain the result returned by MinimalLeastSq diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 04cb490..da88355 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -155,7 +155,7 @@ void JSONReporter::PrintRunData(Run const& run) { } else if(run.report_rms) { out << indent << FormatKV("rms", RoundDouble(run.GetAdjustedCPUTime()*100)) - << "%"; + << '%'; } if (run.bytes_per_second > 0.0) { out << ",\n" << indent -- cgit v1.2.3 From 109f528a4039c298e1f02ff4ce2fd32c552c5a38 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 19:48:53 +0200 Subject: removed functional library not needed --- include/benchmark/benchmark_api.h | 1 - src/complexity.cc | 1 - test/complexity_test.cc | 6 +++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 49167ff..0cb4348 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -152,7 +152,6 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include #include -#include #include "macros.h" diff --git a/src/complexity.cc b/src/complexity.cc index 0d6f90b..1b444ce 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -22,7 +22,6 @@ #include "stat.h" #include #include -#include namespace benchmark { diff --git a/test/complexity_test.cc b/test/complexity_test.cc index ed434fe..03bd2dc 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -153,9 +153,9 @@ void BM_Complexity_O1(benchmark::State& state) { } state.SetComplexityN(state.range_x()); } -BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity([](size_t){return 1.0; }); -BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](size_t){return 1.0; }); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(); std::string big_o_1_test_name = "BM_Complexity_O1_BigO"; std::string rms_o_1_test_name = "BM_Complexity_O1_RMS"; -- cgit v1.2.3 From 3ef63399716c7ac213d4016ab9454422f4f9d6d1 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 20:58:14 +0200 Subject: Update Readme.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5be5153..f34e887 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,14 @@ BENCHMARK(BM_StringCompare) ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(); ``` +The following code will specify asymptotic complexity with a lambda function, +that might be used to customize high-order term calculation. + +```c++ +BENCHMARK(BM_StringCompare)->RangeMultiplier(2) + ->Range(1<<10, 1<<18)->Complexity([](size_t n)->double{return n; }); +``` + ### Templated benchmarks Templated benchmarks work the same way: This example produces and consumes messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the -- cgit v1.2.3 From 22cb9d9ce0ff12219f5ca6c4a28124d11730e66f Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 22:01:31 +0200 Subject: google formated --- include/benchmark/reporter.h | 52 ++++++++++---------- src/complexity.cc | 112 +++++++++++++++++++++---------------------- src/console_reporter.cc | 26 +++++----- src/csv_reporter.cc | 10 ++-- src/json_reporter.cc | 75 +++++++++++++++-------------- 5 files changed, 137 insertions(+), 138 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index f37e0a3..e3a8f57 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -20,7 +20,7 @@ #include #include -#include "benchmark_api.h" // For forward declaration of BenchmarkReporter +#include "benchmark_api.h" // For forward declaration of BenchmarkReporter namespace benchmark { @@ -133,14 +133,14 @@ class BenchmarkReporter { error_stream_ = err; } - std::ostream& GetOutputStream() const { + std::ostream& GetOutputStream() const { return *output_stream_; } std::ostream& GetErrorStream() const { return *error_stream_; } - + virtual ~BenchmarkReporter(); // Write a human readable string to 'out' representing the specified @@ -148,7 +148,7 @@ class BenchmarkReporter { // REQUIRES: 'out' is non-null. static void PrintBasicContext(std::ostream* out, Context const& context); -private: + private: std::ostream* output_stream_; std::ostream* error_stream_; }; @@ -156,61 +156,61 @@ private: // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). class ConsoleReporter : public BenchmarkReporter { -public: + public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); -protected: + protected: virtual void PrintRunData(const Run& report); size_t name_field_width_; }; class JSONReporter : public BenchmarkReporter { -public: + public: JSONReporter() : first_report_(true) {} virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); virtual void Finalize(); -private: + private: void PrintRunData(const Run& report); bool first_report_; }; class CSVReporter : public BenchmarkReporter { -public: + public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); -private: + private: void PrintRunData(const Run& report); }; inline const char* GetTimeUnitString(TimeUnit unit) { switch (unit) { - case kMillisecond: - return "ms"; - case kMicrosecond: - return "us"; - case kNanosecond: - default: - return "ns"; + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + default: + return "ns"; } } inline double GetTimeUnitMultiplier(TimeUnit unit) { switch (unit) { - case kMillisecond: - return 1e3; - case kMicrosecond: - return 1e6; - case kNanosecond: - default: - return 1e9; + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + default: + return 1e9; } } -} // end namespace benchmark -#endif // BENCHMARK_REPORTER_H_ +} // end namespace benchmark +#endif // BENCHMARK_REPORTER_H_ diff --git a/src/complexity.cc b/src/complexity.cc index 1b444ce..0d53ade 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -17,55 +17,55 @@ #include "benchmark/benchmark_api.h" -#include "complexity.h" +#include +#include #include "check.h" +#include "complexity.h" #include "stat.h" -#include -#include namespace benchmark { // Internal function to calculate the different scalability forms BigOFunc* FittingCurve(BigO complexity) { switch (complexity) { - case oN: - return [](size_t n) -> double {return n; }; - case oNSquared: - return [](size_t n) -> double {return n * n; }; - case oNCubed: - return [](size_t n) -> double {return n * n * n; }; - case oLogN: - return [](size_t n) {return log2(n); }; - case oNLogN: - return [](size_t n) {return n * log2(n); }; - case o1: - default: - return [](size_t) {return 1.0; }; + case oN: + return [](size_t n) -> double { return n; }; + case oNSquared: + return [](size_t n) -> double { return n * n; }; + case oNCubed: + return [](size_t n) -> double { return n * n * n; }; + case oLogN: + return [](size_t n) { return log2(n); }; + case oNLogN: + return [](size_t n) { return n * log2(n); }; + case o1: + default: + return [](size_t) { return 1.0; }; } } // Function to return an string for the calculated complexity std::string GetBigOString(BigO complexity) { switch (complexity) { - case oN: - return "N"; - case oNSquared: - return "N^2"; - case oNCubed: - return "N^3"; - case oLogN: - return "lgN"; - case oNLogN: - return "NlgN"; - case o1: - return "(1)"; - default: - return "f(N)"; + case oN: + return "N"; + case oNSquared: + return "N^2"; + case oNCubed: + return "N^3"; + case oLogN: + return "lgN"; + case oNLogN: + return "NlgN"; + case o1: + return "(1)"; + default: + return "f(N)"; } } -// Find the coefficient for the high-order term in the running time, by -// minimizing the sum of squares of relative error, for the fitting curve +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error, for the fitting curve // given by the lambda expresion. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. @@ -122,14 +122,14 @@ LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { CHECK_EQ(n.size(), time.size()); - CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given + CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two + // benchmark runs are given CHECK_NE(complexity, oNone); LeastSq best_fit; - if(complexity == oAuto) { - std::vector fit_curves = { - oLogN, oN, oNLogN, oNSquared, oNCubed }; + if (complexity == oAuto) { + std::vector fit_curves = {oLogN, oN, oNLogN, oNSquared, oNCubed}; // Take o1 as default best fitting curve best_fit = MinimalLeastSq(n, time, FittingCurve(o1)); @@ -152,14 +152,13 @@ LeastSq MinimalLeastSq(const std::vector& n, } std::vector ComputeStats( - const std::vector& reports) -{ + const std::vector& reports) { typedef BenchmarkReporter::Run Run; std::vector results; - auto error_count = std::count_if( - reports.begin(), reports.end(), - [](Run const& run) {return run.error_occurred;}); + auto error_count = + std::count_if(reports.begin(), reports.end(), + [](Run const& run) { return run.error_occurred; }); if (reports.size() - error_count < 2) { // We don't report aggregated data if there was a single run. @@ -178,12 +177,11 @@ std::vector ComputeStats( for (Run const& run : reports) { CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); CHECK_EQ(run_iterations, run.iterations); - if (run.error_occurred) - continue; + if (run.error_occurred) continue; real_accumulated_time_stat += - Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); + Stat1_d(run.real_accumulated_time / run.iterations, run.iterations); cpu_accumulated_time_stat += - Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); + Stat1_d(run.cpu_accumulated_time / run.iterations, run.iterations); items_per_second_stat += Stat1_d(run.items_per_second, run.iterations); bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations); } @@ -192,10 +190,10 @@ std::vector ComputeStats( Run mean_data; mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; mean_data.iterations = run_iterations; - mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * - run_iterations; - mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * - run_iterations; + mean_data.real_accumulated_time = + real_accumulated_time_stat.Mean() * run_iterations; + mean_data.cpu_accumulated_time = + cpu_accumulated_time_stat.Mean() * run_iterations; mean_data.bytes_per_second = bytes_per_second_stat.Mean(); mean_data.items_per_second = items_per_second_stat.Mean(); @@ -212,10 +210,8 @@ std::vector ComputeStats( stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; stddev_data.report_label = mean_data.report_label; stddev_data.iterations = 0; - stddev_data.real_accumulated_time = - real_accumulated_time_stat.StdDev(); - stddev_data.cpu_accumulated_time = - cpu_accumulated_time_stat.StdDev(); + stddev_data.real_accumulated_time = real_accumulated_time_stat.StdDev(); + stddev_data.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); stddev_data.items_per_second = items_per_second_stat.StdDev(); @@ -225,8 +221,7 @@ std::vector ComputeStats( } std::vector ComputeBigO( - const std::vector& reports) -{ + const std::vector& reports) { typedef BenchmarkReporter::Run Run; std::vector results; @@ -240,8 +235,8 @@ std::vector ComputeBigO( // Populate the accumulators. for (const Run& run : reports) { n.push_back(run.complexity_n); - real_time.push_back(run.real_accumulated_time/run.iterations); - cpu_time.push_back(run.cpu_accumulated_time/run.iterations); + real_time.push_back(run.real_accumulated_time / run.iterations); + cpu_time.push_back(run.cpu_accumulated_time / run.iterations); } LeastSq result_cpu; @@ -254,7 +249,8 @@ std::vector ComputeBigO( result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); } - std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); + std::string benchmark_name = + reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 2783097..080c324 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -15,9 +15,9 @@ #include "benchmark/reporter.h" #include "complexity.h" +#include #include #include -#include #include #include #include @@ -62,8 +62,8 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { void ConsoleReporter::PrintRunData(const Run& result) { auto& Out = GetOutputStream(); - auto name_color = (result.report_big_o || result.report_rms) - ? COLOR_BLUE : COLOR_GREEN; + auto name_color = + (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; ColorPrintf(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name.c_str()); @@ -84,25 +84,25 @@ void ConsoleReporter::PrintRunData(const Run& result) { if (result.items_per_second > 0) { items = StrCat(" ", HumanReadableNumber(result.items_per_second), " items/s"); - } + } const double real_time = result.GetAdjustedRealTime(); const double cpu_time = result.GetAdjustedCPUTime(); - if(result.report_big_o) { + if (result.report_big_o) { std::string big_o = GetBigOString(result.complexity); - ColorPrintf(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", - real_time, big_o.c_str(), cpu_time, big_o.c_str()); - } else if(result.report_rms) { - ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", - real_time * 100, cpu_time * 100); + ColorPrintf(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", real_time, + big_o.c_str(), cpu_time, big_o.c_str()); + } else if (result.report_rms) { + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", real_time * 100, + cpu_time * 100); } else { const char* timeLabel = GetTimeUnitString(result.time_unit); - ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", - real_time, timeLabel, cpu_time, timeLabel); + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", real_time, timeLabel, + cpu_time, timeLabel); } - if(!result.report_big_o && !result.report_rms) { + if (!result.report_big_o && !result.report_rms) { ColorPrintf(Out, COLOR_CYAN, "%10lld", result.iterations); } diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 775a46c..7bc7ef3 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -15,8 +15,8 @@ #include "benchmark/reporter.h" #include "complexity.h" -#include #include +#include #include #include #include @@ -80,7 +80,7 @@ void CSVReporter::PrintRunData(const Run & run) { } // Do not print iteration on bigO and RMS report - if(!run.report_big_o && !run.report_rms) { + if (!run.report_big_o && !run.report_rms) { Out << run.iterations; } Out << ","; @@ -89,9 +89,9 @@ void CSVReporter::PrintRunData(const Run & run) { Out << run.GetAdjustedCPUTime() << ","; // Do not print timeLabel on bigO and RMS report - if(run.report_big_o) { + if (run.report_big_o) { Out << GetBigOString(run.complexity); - } else if(!run.report_rms){ + } else if (!run.report_rms) { Out << GetTimeUnitString(run.time_unit); } Out << ","; @@ -111,7 +111,7 @@ void CSVReporter::PrintRunData(const Run & run) { ReplaceAll(&label, "\"", "\"\""); Out << "\"" << label << "\""; } - Out << ",,"; // for error_occurred and error_message + Out << ",,"; // for error_occurred and error_message Out << '\n'; } diff --git a/src/json_reporter.cc b/src/json_reporter.cc index da88355..485d305 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -15,8 +15,8 @@ #include "benchmark/reporter.h" #include "complexity.h" -#include #include +#include #include #include #include @@ -100,24 +100,24 @@ void JSONReporter::ReportRuns(std::vector const& reports) { first_report_ = false; for (auto it = reports.begin(); it != reports.end(); ++it) { - out << indent << "{\n"; - PrintRunData(*it); - out << indent << '}'; - auto it_cp = it; - if (++it_cp != reports.end()) { - out << ",\n"; - } + out << indent << "{\n"; + PrintRunData(*it); + out << indent << '}'; + auto it_cp = it; + if (++it_cp != reports.end()) { + out << ",\n"; + } } } void JSONReporter::Finalize() { - // Close the list of benchmarks and the top level object. - GetOutputStream() << "\n ]\n}\n"; + // Close the list of benchmarks and the top level object. + GetOutputStream() << "\n ]\n}\n"; } void JSONReporter::PrintRunData(Run const& run) { - std::string indent(6, ' '); - std::ostream& out = GetOutputStream(); + std::string indent(6, ' '); + std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name) << ",\n"; @@ -129,7 +129,7 @@ void JSONReporter::PrintRunData(Run const& run) { << FormatKV("error_message", run.error_message) << ",\n"; } - if(!run.report_big_o && !run.report_rms) { + if (!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; @@ -140,14 +140,14 @@ void JSONReporter::PrintRunData(Run const& run) { << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime())); out << ",\n" << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); - } else if(run.report_big_o) { - out << indent - << FormatKV("cpu_coefficient", RoundDouble(run.GetAdjustedCPUTime())) - << ",\n"; - out << indent - << FormatKV("real_coefficient", RoundDouble(run.GetAdjustedRealTime())) - << ",\n"; - out << indent + } else if (run.report_big_o) { + out << indent + << FormatKV("cpu_coefficient", RoundDouble(run.GetAdjustedCPUTime())) + << ",\n"; + out << indent + << FormatKV("real_coefficient", RoundDouble(run.GetAdjustedRealTime())) + << ",\n"; + out << indent << FormatKV("big_o", GetBigOString(run.complexity)) << ",\n"; out << indent @@ -156,20 +156,23 @@ void JSONReporter::PrintRunData(Run const& run) { out << indent << FormatKV("rms", RoundDouble(run.GetAdjustedCPUTime()*100)) << '%'; - } - if (run.bytes_per_second > 0.0) { - out << ",\n" << indent - << FormatKV("bytes_per_second", RoundDouble(run.bytes_per_second)); - } - if (run.items_per_second > 0.0) { - out << ",\n" << indent - << FormatKV("items_per_second", RoundDouble(run.items_per_second)); - } - if (!run.report_label.empty()) { - out << ",\n" << indent - << FormatKV("label", run.report_label); - } - out << '\n'; + } + if (run.bytes_per_second > 0.0) { + out << ",\n" + << indent + << FormatKV("bytes_per_second", RoundDouble(run.bytes_per_second)); + } + if (run.items_per_second > 0.0) { + out << ",\n" + << indent + << FormatKV("items_per_second", RoundDouble(run.items_per_second)); + } + if (!run.report_label.empty()) { + out << ",\n" + << indent + << FormatKV("label", run.report_label); + } + out << '\n'; } -} // end namespace benchmark +} // end namespace benchmark -- cgit v1.2.3 From 240ba4e64eb46e1f5acbafaadae34c2b2ca701eb Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 22:21:52 +0200 Subject: changed BigOFunc argument from size_t to int --- README.md | 2 +- include/benchmark/benchmark_api.h | 2 +- src/complexity.cc | 12 ++++++------ test/complexity_test.cc | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f34e887..e30052d 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ that might be used to customize high-order term calculation. ```c++ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) - ->Range(1<<10, 1<<18)->Complexity([](size_t n)->double{return n; }); + ->Range(1<<10, 1<<18)->Complexity([](int n)->double{return n; }); ``` ### Templated benchmarks diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 0cb4348..34cd0b5 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -253,7 +253,7 @@ enum BigO { // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(size_t); +typedef double(BigOFunc)(int); // State is passed to a running Benchmark and contains state for the // benchmark to use. diff --git a/src/complexity.cc b/src/complexity.cc index 0d53ade..e25aa3c 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -29,18 +29,18 @@ namespace benchmark { BigOFunc* FittingCurve(BigO complexity) { switch (complexity) { case oN: - return [](size_t n) -> double { return n; }; + return [](int n) -> double { return n; }; case oNSquared: - return [](size_t n) -> double { return n * n; }; + return [](int n) -> double { return n * n; }; case oNCubed: - return [](size_t n) -> double { return n * n * n; }; + return [](int n) -> double { return n * n * n; }; case oLogN: - return [](size_t n) { return log2(n); }; + return [](int n) { return log2(n); }; case oNLogN: - return [](size_t n) { return n * log2(n); }; + return [](int n) { return n * log2(n); }; case o1: default: - return [](size_t) { return 1.0; }; + return [](int) { return 1.0; }; } } diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 03bd2dc..662b627 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -154,7 +154,7 @@ void BM_Complexity_O1(benchmark::State& state) { state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](size_t){return 1.0; }); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](int){return 1.0; }); BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(); std::string big_o_1_test_name = "BM_Complexity_O1_BigO"; @@ -192,7 +192,7 @@ void BM_Complexity_O_N(benchmark::State& state) { state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); -BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) -> double{return n; }); +BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) -> double{return n; }); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); std::string big_o_n_test_name = "BM_Complexity_O_N_BigO"; @@ -220,7 +220,7 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](size_t n) {return n * log2(n); }); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) {return n * log2(n); }); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); std::string big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; -- cgit v1.2.3 From 1a633969b31b2a486bdfae80576c53f40293281e Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 22:23:39 +0200 Subject: changed BigO logic order --- src/complexity.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index e25aa3c..24f1cf4 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -242,12 +242,12 @@ std::vector ComputeBigO( LeastSq result_cpu; LeastSq result_real; - if (reports[0].complexity != oLambda) { - result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); - } else { + if (reports[0].complexity == oLambda) { result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); + } else { + result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); + result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); } std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); -- cgit v1.2.3 From 8ba94b4c1842d9424c592258a6dfc9beea4912c8 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 22:40:21 +0200 Subject: changed global string to const char * --- test/complexity_test.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 662b627..ee24202 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -157,10 +157,10 @@ BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](int){return 1.0; }); BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(); -std::string big_o_1_test_name = "BM_Complexity_O1_BigO"; -std::string rms_o_1_test_name = "BM_Complexity_O1_RMS"; -std::string enum_auto_big_o_1 = "\\([0-9]+\\)"; -std::string lambda_big_o_1 = "f\\(N\\)"; +const char* big_o_1_test_name = "BM_Complexity_O1_BigO"; +const char* rms_o_1_test_name = "BM_Complexity_O1_RMS"; +const char* enum_auto_big_o_1 = "\\([0-9]+\\)"; +const char* lambda_big_o_1 = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, @@ -195,10 +195,10 @@ BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Com BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) -> double{return n; }); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); -std::string big_o_n_test_name = "BM_Complexity_O_N_BigO"; -std::string rms_o_n_test_name = "BM_Complexity_O_N_RMS"; -std::string enum_auto_big_o_n = "N"; -std::string lambda_big_o_n = "f\\(N\\)"; +const char* big_o_n_test_name = "BM_Complexity_O_N_BigO"; +const char* rms_o_n_test_name = "BM_Complexity_O_N_RMS"; +const char* enum_auto_big_o_n = "N"; +const char* lambda_big_o_n = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, @@ -223,10 +223,10 @@ BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) {return n * log2(n); }); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); -std::string big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; -std::string rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; -std::string enum_auto_big_o_n_lg_n = "NlgN"; -std::string lambda_big_o_n_lg_n = "f\\(N\\)"; +const char* big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; +const char* rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; +const char* enum_auto_big_o_n_lg_n = "NlgN"; +const char* lambda_big_o_n_lg_n = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, -- cgit v1.2.3 From 2859ae93949a7a3415082e65001f25e8e5e78284 Mon Sep 17 00:00:00 2001 From: Ismael Date: Thu, 2 Jun 2016 23:27:29 +0200 Subject: changed complexity_n to int and fix some whitespaces --- include/benchmark/benchmark_api.h | 4 ++-- include/benchmark/reporter.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 34cd0b5..f38dc97 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -363,7 +363,7 @@ public: // family benchmark, then current benchmark will be part of the computation and complexity_n will // represent the length of N. BENCHMARK_ALWAYS_INLINE - void SetComplexityN(size_t complexity_n) { + void SetComplexityN(int complexity_n) { complexity_n_ = complexity_n; } @@ -444,7 +444,7 @@ private: size_t bytes_processed_; size_t items_processed_; - size_t complexity_n_; + int complexity_n_; public: // FIXME: Make this private somehow. diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index e3a8f57..22c97a0 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -87,7 +87,7 @@ class BenchmarkReporter { // Keep track of arguments to compute asymptotic complexity BigO complexity; BigOFunc* complexity_lambda; - size_t complexity_n; + int complexity_n; // Inform print function whether the current run is a complexity report bool report_big_o; @@ -133,14 +133,14 @@ class BenchmarkReporter { error_stream_ = err; } - std::ostream& GetOutputStream() const { + std::ostream& GetOutputStream() const { return *output_stream_; } std::ostream& GetErrorStream() const { return *error_stream_; } - + virtual ~BenchmarkReporter(); // Write a human readable string to 'out' representing the specified -- cgit v1.2.3 From e49814316891c8d6e125da5d3f72c7bc45f31bf4 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 2 Jun 2016 14:37:14 -0700 Subject: fix warning on loss of integer precision --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 0065433..f6c4fc2 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -130,7 +130,7 @@ struct ThreadStats { ThreadStats() : bytes_processed(0), items_processed(0), complexity_n(0) {} int64_t bytes_processed; int64_t items_processed; - size_t complexity_n; + int complexity_n; }; // Timer management class -- cgit v1.2.3 From 3fdd76bd14ff122c6881d7f15ec5cb2629241e7a Mon Sep 17 00:00:00 2001 From: Ismael Date: Fri, 3 Jun 2016 18:33:17 +0200 Subject: fix issue 235 (#236) --- src/complexity.cc | 4 ++-- test/complexity_test.cc | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index 24f1cf4..93833f9 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -35,9 +35,9 @@ BigOFunc* FittingCurve(BigO complexity) { case oNCubed: return [](int n) -> double { return n * n * n; }; case oLogN: - return [](int n) { return log2(n); }; + return [](int n) { return std::log2(n); }; case oNLogN: - return [](int n) { return n * log2(n); }; + return [](int n) { return n * std::log2(n); }; case o1: default: return [](int) { return 1.0; }; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index ee24202..8ab88f9 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -10,6 +10,7 @@ #include #include #include +#include namespace { @@ -220,7 +221,7 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { state.SetComplexityN(state.range_x()); } BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); -BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) {return n * log2(n); }); +BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) {return n * std::log2(n); }); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); const char* big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; -- cgit v1.2.3 From 1d53e5e0d8d0c5b69fc19d7e95dda2c8385d03f4 Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 20 Jun 2016 12:15:09 -0400 Subject: Clarified output formats (#241) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e30052d..498ed61 100644 --- a/README.md +++ b/README.md @@ -382,7 +382,7 @@ static void BM_test(benchmark::State& state) { ## Output Formats The library supports multiple output formats. Use the -`--benchmark_format=` flag to set the format type. `tabular` is +`--benchmark_format=` flag to set the format type. `tabular` is the default format. The Tabular format is intended to be a human readable format. By default -- cgit v1.2.3 From d147797aaccb71f44d7ee3df94dcf14f8d713247 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 27 Jun 2016 13:24:13 -0500 Subject: Add state.SetComplexityN to docs. (#245) Add `state.SetComplexityN(state.range_x());` to documentation so that complexity is calculated correctly. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 498ed61..e1ac17a 100644 --- a/README.md +++ b/README.md @@ -127,8 +127,10 @@ running time and the normalized root-mean square error of string comparison. static void BM_StringCompare(benchmark::State& state) { std::string s1(state.range_x(), '-'); std::string s2(state.range_x(), '-'); - while (state.KeepRunning()) + while (state.KeepRunning()) { benchmark::DoNotOptimize(s1.compare(s2)); + } + state.SetComplexityN(state.range_x()); } BENCHMARK(BM_StringCompare) ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN); -- cgit v1.2.3 From a0783fd7f629789cd406d74392427a73a692bf57 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 27 Jun 2016 20:25:43 +0200 Subject: fix 244 (#247) --- src/benchmark.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchmark.cc b/src/benchmark.cc index f6c4fc2..cb8e132 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -549,6 +549,7 @@ void BenchmarkImp::RangePair(int lo1, int hi1, int lo2, int hi2) { } void BenchmarkImp::RangeMultiplier(int multiplier) { + CHECK(multiplier > 1); range_multiplier_ = multiplier; } -- cgit v1.2.3 From 885ca41cf835313eca052ad112608631685ae6f2 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 27 Jun 2016 20:26:23 +0200 Subject: added check for SetComplexityN (#248) --- src/complexity.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/complexity.cc b/src/complexity.cc index 93833f9..b42bd38 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -234,6 +234,7 @@ std::vector ComputeBigO( // Populate the accumulators. for (const Run& run : reports) { + CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; n.push_back(run.complexity_n); real_time.push_back(run.real_accumulated_time / run.iterations); cpu_time.push_back(run.cpu_accumulated_time / run.iterations); -- cgit v1.2.3 From 2149577f892116d4080d16fbf0b0455b1026b219 Mon Sep 17 00:00:00 2001 From: Steve Downey Date: Wed, 6 Jul 2016 15:36:56 -0400 Subject: Add export linker flags policy to cmake (#251) Add policy CMP0056, which honors the link flags in try_compile and try_run. This allows for building against non-system libc++ by providing the correct -L and rpath options in a containing project. For example: set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L ${LLVM_ROOT}/lib -l c++ -l c++abi") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath,${LLVM_ROOT}/lib") --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f340fb3..a1251e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,7 @@ project (benchmark) foreach(p CMP0054 # CMake 3.1 + CMP0056 # export EXE_LINKER_FLAGS to try_run ) if(POLICY ${p}) cmake_policy(SET ${p} NEW) -- cgit v1.2.3 From 7e40ff9e35699ea14a6addd2ce20cd23be519430 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 11 Jul 2016 14:58:50 -0600 Subject: Provide a better implementation of DoNotOptimize(...). This implementation is less likely to ICE compilers, and is more correct. It also acts as a memory barrier which will help prevent writes to global memory from being optimized away. --- README.md | 45 ++++++++++++++++++++++++++++++++++++++- include/benchmark/benchmark_api.h | 19 ++++++++--------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e1ac17a..a0bcc61 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,8 @@ BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime(); ### Preventing optimisation To prevent a value or expression from being optimized away by the compiler -the `benchmark::DoNotOptimize(...)` function can be used. +the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()` +functions can be used. ```c++ static void BM_test(benchmark::State& state) { @@ -292,6 +293,48 @@ static void BM_test(benchmark::State& state) { } ``` +`DoNotOptimize()` forces the *result* of `` to be stored in either +memory or a register. For GNU based compilers it acts as read/write barrier +for global memory. More specifically it forces the compiler to flush pending +writes to memory and reload any other values as necessary. + +Note that `DoNotOptimize()` does not prevent optimizations on `` +in any way. `` may even be removed entirely when the result is already +known. For example: + +```c++ + /* Example 1: `` is removed entirely. */ + int foo(int x) { return x + 42; } + while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42); + + /* Example 2: Result of '' is only reused */ + int bar(int) __attribute__((const)); + while (...) DoNotOptimize(bar(0)); // Optimized to: + // int __result__ = bar(0); + // while (...) DoNotOptimize(__result__); +``` + +The second tool for preventing optimizations is `ClobberMemory()`. In essence +`ClobberMemory()` forces the compiler to perform all pending writes to global +memory. Memory managed by block scope objects must be "escaped" using +`DoNotOptimize(...)` before it can be clobbered. In the below example +`ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized +away. + +```c++ +static void BM_vector_push_back(benchmark::State& state) { + while (state.KeepRunning()) { + std::vector v; + v.reserve(1); + benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered. + v.push_back(42); + benchmark::ClobberMemory(); // Force 42 to be written to memory. + } +} +``` + +Note that `ClobberMemory()` is only available for GNU based compilers. + ### Set time unit manually If a benchmark runs a few milliseconds it may be hard to visually compare the measured times, since the output data is given in nanoseconds per default. In diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index f38dc97..664ca2a 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -207,25 +207,24 @@ Benchmark* RegisterBenchmarkInternal(Benchmark*); // The DoNotOptimize(...) function can be used to prevent a value or // expression from being optimized away by the compiler. This function is -// intented to add little to no overhead. -// See: http://stackoverflow.com/questions/28287064 +// intended to add little to no overhead. +// See: https://youtu.be/nXaxk27zwlk?t=2441 #if defined(__GNUC__) -// TODO(ericwf): Clang has a bug where it tries to always use a register -// even if value must be stored in memory. This causes codegen to fail. -// To work around this we remove the "r" modifier so the operand is always -// loaded into memory. -// GCC also has a bug where it complains about inconsistent operand constraints -// when "+rm" is used for a type larger than can fit in a register or two. -// For now force the operand to memory for both GCC and Clang. template inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : "+m" (const_cast(value))); + asm volatile("" : : "g"(value) : "memory"); +} +// Force the compiler to flush pending writes to global memory. Acts as an +// effective read/write barrier +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { + asm volatile("" : : : "memory"); } #else template inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); } +// FIXME Add ClobberMemory() for non-gnu compilers #endif // TimeUnit is passed to a benchmark in order to specify the order of magnitude -- cgit v1.2.3 From b805b7c6e02d0a1d1709f14295c0b05b988a5668 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Tue, 12 Jul 2016 12:40:09 -0700 Subject: Add missing `volatile`s to 32-bit ARM cycleclock assembler. (#253) Without these, clang reorders these instructions as if they were regular loads/stores which causes SIGILL from the kernel because it performs all the loads before it starts testing the values. --- src/cycleclock.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cycleclock.h b/src/cycleclock.h index 3110804..e4825d4 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -113,11 +113,11 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { uint32_t pmuseren; uint32_t pmcntenset; // Read the user mode perf monitor counter access permissions. - asm("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); + asm volatile("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); if (pmuseren & 1) { // Allows reading perfmon counters for user mode code. - asm("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); + asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); if (pmcntenset & 0x80000000ul) { // Is it counting? - asm("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); // The counter is set up to count every 64th cycle return static_cast(pmccntr) * 64; // Should optimize to << 6 } -- cgit v1.2.3 From ebd37b191c757cd0b20c9c8517e38d1e1ac45069 Mon Sep 17 00:00:00 2001 From: Sven Date: Wed, 13 Jul 2016 22:00:32 +0200 Subject: set cpuinfo_num_cpus on Windows (#254) --- src/sysinfo.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index e10e19d..3a5d942 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -239,6 +239,7 @@ void InitializeSystemInfo() { } // TODO: also figure out cpuinfo_num_cpus + #elif defined BENCHMARK_OS_WINDOWS // In NT, read MHz from the registry. If we fail to do so or we're in win9x // then make a crude estimate. @@ -251,7 +252,10 @@ void InitializeSystemInfo() { cpuinfo_cycles_per_second = static_cast((int64_t)data * (int64_t)(1000 * 1000)); // was mhz else cpuinfo_cycles_per_second = static_cast(EstimateCyclesPerSecond()); -// TODO: also figure out cpuinfo_num_cpus + + SYSTEM_INFO sysinfo = { 0 }; + GetSystemInfo(&sysinfo); + cpuinfo_num_cpus = sysinfo.dwNumberOfProcessors; // number of logical processors in the current group #elif defined BENCHMARK_OS_MACOSX // returning "mach time units" per second. the current number of elapsed -- cgit v1.2.3