From df7bd92afd9e5b803cb29d92f4f545291a5f045e Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 29 Aug 2016 12:43:55 -0600 Subject: Add warning for large benchmark families (#279) * Add warning for large benchmark families * reflow comment --- src/benchmark.cc | 112 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 2306c2d..bfac28c 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -115,6 +115,9 @@ bool IsZero(double n) { // For non-dense Range, intermediate values are powers of kRangeMultiplier. static const int kRangeMultiplier = 8; +// The size of a benchmark family determines is the number of inputs to repeat +// the benchmark on. If this is "large" then warn the user during configuration. +static const size_t kMaxFamilySize = 100; static const size_t kMaxIterations = 1000000000; bool running_benchmark = false; @@ -353,7 +356,8 @@ class BenchmarkFamilies { // Extract the list of benchmark instances that match the specified // regular expression. bool FindBenchmarks(const std::string& re, - std::vector* benchmarks); + std::vector* benchmarks, + std::ostream* Err); private: BenchmarkFamilies() {} @@ -424,18 +428,20 @@ size_t BenchmarkFamilies::AddBenchmark(std::unique_ptr family) { bool BenchmarkFamilies::FindBenchmarks( const std::string& spec, - std::vector* benchmarks) { + std::vector* benchmarks, + std::ostream* ErrStream) { + CHECK(ErrStream); + auto& Err = *ErrStream; // Make regular expression out of command-line flag std::string error_msg; Regex re; if (!re.Init(spec, &error_msg)) { - std::cerr << "Could not compile benchmark re: " << error_msg << std::endl; + Err << "Could not compile benchmark re: " << error_msg << std::endl; return false; } // Special list of thread counts to use when none are specified - std::vector one_thread; - one_thread.push_back(1); + const std::vector one_thread = {1}; MutexLock l(mutex_); for (std::unique_ptr& bench_family : families_) { @@ -446,12 +452,23 @@ bool BenchmarkFamilies::FindBenchmarks( if (family->ArgsCnt() == -1) { family->Args({}); } - - for (auto const& args : family->args_) { - const std::vector* thread_counts = + const std::vector* thread_counts = (family->thread_counts_.empty() ? &one_thread - : &family->thread_counts_); + : &static_cast&>(family->thread_counts_)); + const size_t family_size = family->args_.size() * thread_counts->size(); + // The benchmark will be run at least 'family_size' different inputs. + // If 'family_size' is very large warn the user. + if (family_size > kMaxFamilySize) { + Err << "The number of inputs is very large. " << family->name_ + << " will be repeated at least " << family_size << " times.\n"; + } + // reserve in the special case the regex ".", since we know the final + // family size. + if (spec == ".") + benchmarks->reserve(family_size); + + for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { Benchmark::Instance instance; @@ -493,8 +510,8 @@ bool BenchmarkFamilies::FindBenchmarks( } if (re.Match(instance.name)) { - instance.last_benchmark_instance = (args == family->args_.back()); - benchmarks->push_back(instance); + instance.last_benchmark_instance = (&args == &family->args_.back()); + benchmarks->push_back(std::move(instance)); } } } @@ -560,7 +577,7 @@ void BenchmarkImp::Ranges(const std::vector>& ranges) { tmp.reserve(arglists.size()); for (std::size_t j = 0; j < arglists.size(); j++) { - tmp.push_back(arglists[j][ctr[j]]); + tmp.push_back(arglists[j].at(ctr[j])); } args_.push_back(std::move(tmp)); @@ -1093,47 +1110,52 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, if (spec.empty() || spec == "all") spec = "."; // Regexp that matches all benchmarks - std::vector benchmarks; - auto families = internal::BenchmarkFamilies::GetInstance(); - if (!families->FindBenchmarks(spec, &benchmarks)) return 0; - - if (FLAGS_benchmark_list_tests) { - for (auto const& benchmark : benchmarks) - std::cout << benchmark.name << "\n"; - } else { - // Setup the reporters - std::ofstream output_file; - std::unique_ptr default_console_reporter; - std::unique_ptr default_file_reporter; - if (!console_reporter) { - auto output_opts = FLAGS_color_print ? ConsoleReporter::OO_Color - : ConsoleReporter::OO_None; - default_console_reporter = internal::CreateReporter( + // Setup the reporters + std::ofstream output_file; + std::unique_ptr default_console_reporter; + std::unique_ptr default_file_reporter; + if (!console_reporter) { + auto output_opts = FLAGS_color_print ? ConsoleReporter::OO_Color + : ConsoleReporter::OO_None; + default_console_reporter = internal::CreateReporter( FLAGS_benchmark_format, output_opts); - console_reporter = default_console_reporter.get(); - } - std::string const& fname = FLAGS_benchmark_out; - if (fname == "" && file_reporter) { - std::cerr << "A custom file reporter was provided but " + console_reporter = default_console_reporter.get(); + } + auto& Out = console_reporter->GetOutputStream(); + auto& Err = console_reporter->GetErrorStream(); + + std::string const& fname = FLAGS_benchmark_out; + if (fname == "" && file_reporter) { + Err << "A custom file reporter was provided but " "--benchmark_out= was not specified." << std::endl; + std::exit(1); + } + if (fname != "") { + output_file.open(fname); + if (!output_file.is_open()) { + Err << "invalid file name: '" << fname << std::endl; std::exit(1); } - if (fname != "") { - output_file.open(fname); - if (!output_file.is_open()) { - std::cerr << "invalid file name: '" << fname << std::endl; - std::exit(1); - } - if (!file_reporter) { - default_file_reporter = internal::CreateReporter( + if (!file_reporter) { + default_file_reporter = internal::CreateReporter( FLAGS_benchmark_out_format, ConsoleReporter::OO_None); - file_reporter = default_file_reporter.get(); - } - file_reporter->SetOutputStream(&output_file); - file_reporter->SetErrorStream(&output_file); + file_reporter = default_file_reporter.get(); } + file_reporter->SetOutputStream(&output_file); + file_reporter->SetErrorStream(&output_file); + } + + std::vector benchmarks; + auto families = internal::BenchmarkFamilies::GetInstance(); + if (!families->FindBenchmarks(spec, &benchmarks, &Err)) return 0; + + if (FLAGS_benchmark_list_tests) { + for (auto const& benchmark : benchmarks) + Out << benchmark.name << "\n"; + } else { internal::RunMatchingBenchmarks(benchmarks, console_reporter, file_reporter); } + return benchmarks.size(); } -- cgit v1.2.3