diff options
author | Haibo Huang <hhb@google.com> | 2019-09-06 17:31:57 -0700 |
---|---|---|
committer | Haibo Huang <hhb@google.com> | 2019-09-06 17:31:57 -0700 |
commit | 64bb0ff47f741d206b9676cf69e1435cc50ad512 (patch) | |
tree | 75b34f607b8cbe846843582a101f502829beb072 | |
parent | 984fd060925a9c1ba5a3944e9c116464771f84c1 (diff) | |
parent | 7ee72863fdb1ccb2af5a011250b56af3f49b7511 (diff) | |
download | google-benchmark-64bb0ff47f741d206b9676cf69e1435cc50ad512.tar.gz |
Upgrade google-benchmark to 7ee72863fdb1ccb2af5a011250b56af3f49b7511ndk-sysroot-r21
Test: None
Change-Id: I04175dd39de9d9768851642576873359befbac4e
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | AUTHORS | 1 | ||||
-rw-r--r-- | CONTRIBUTORS | 1 | ||||
-rw-r--r-- | METADATA | 6 | ||||
-rw-r--r-- | README.md | 30 | ||||
-rw-r--r-- | include/benchmark/benchmark.h | 5 | ||||
-rw-r--r-- | src/benchmark.cc | 114 | ||||
-rw-r--r-- | src/commandlineflags.h | 11 | ||||
-rw-r--r-- | src/console_reporter.cc | 22 | ||||
-rw-r--r-- | src/counter.cc | 4 | ||||
-rw-r--r-- | test/user_counters_test.cc | 83 |
11 files changed, 192 insertions, 86 deletions
@@ -8,6 +8,7 @@ !/cmake/*.cmake !/test/AssemblyTests.cmake *~ +*.swp *.pyc __pycache__ @@ -44,6 +44,7 @@ Ori Livneh <ori.livneh@gmail.com> Paul Redmond <paul.redmond@gmail.com> Radoslav Yovchev <radoslav.tm@gmail.com> Roman Lebedev <lebedev.ri@gmail.com> +Sayan Bhattacharjee <aero.sayan@gmail.com> Shuo Chen <chenshuo@chenshuo.com> Steinar H. Gunderson <sgunderson@bigfoot.com> Stripe, Inc. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 167165f..5be6dc4 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -65,6 +65,7 @@ Raul Marin <rmrodriguez@cartodb.com> Ray Glover <ray.glover@uk.ibm.com> Robert Guo <robert.guo@mongodb.com> Roman Lebedev <lebedev.ri@gmail.com> +Sayan Bhattacharjee <aero.sayan@gmail.com> Shuo Chen <chenshuo@chenshuo.com> Tobias Ulvgård <tobias.ulvgard@dirac.se> Tom Madams <tom.ej.madams@gmail.com> <tmadams@google.com> @@ -9,10 +9,10 @@ third_party { type: GIT value: "https://github.com/google/benchmark.git" } - version: "c408461983dd3adf49d450d7db926fc46f1d99a0" + version: "7ee72863fdb1ccb2af5a011250b56af3f49b7511" last_upgrade_date { year: 2019 - month: 8 - day: 7 + month: 9 + day: 6 } } @@ -49,7 +49,7 @@ The following minimum versions are required to build the library: * GCC 4.8 * Clang 3.4 -* Visual Studio 2013 +* Visual Studio 14 2015 * Intel 2015 Update 1 ## Installation @@ -422,6 +422,22 @@ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); ``` Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. +The preceding code shows a method of defining a sparse range. The following +example shows a method of defining a dense range. It is then used to benchmark +the performance of `std::vector` initialization for uniformly increasing sizes. + +```c++ +static void BM_DenseRange(benchmark::State& state) { + for(auto _ : state) { + std::vector<int> v(state.range(0), state.range(0)); + benchmark::DoNotOptimize(v.data()); + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128); +``` +Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ]. + You might have a benchmark that depends on two or more inputs. For example, the following code defines a family of benchmarks for measuring the speed of set insertion. @@ -662,9 +678,9 @@ the resulting sum is the value which will be shown for the benchmark. The `Counter` constructor accepts three parameters: the value as a `double` ; a bit flag which allows you to show counters as rates, and/or as per-thread -iteration, and/or as per-thread averages, and/or iteration invariants; -and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, -`benchmark::Counter::OneK::kIs1000`), or 1024 +iteration, and/or as per-thread averages, and/or iteration invariants, +and/or finally inverting the result; and a flag specifying the 'unit' - i.e. +is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 (`benchmark::Counter::OneK::kIs1024`)? ```c++ @@ -673,8 +689,14 @@ and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, // Set the counter as a rate. It will be presented divided // by the duration of the benchmark. + // Meaning: per one second, how many 'foo's are processed? state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); + // Set the counter as a rate. It will be presented divided + // by the duration of the benchmark, and the result inverted. + // Meaning: how many seconds it takes to process one 'foo'? + state.counters["FooInvRate"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); + // Set the counter as a thread-average quantity. It will // be presented divided by the number of threads. state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 856605f..144e212 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -368,7 +368,10 @@ class Counter { // It will be presented divided by the number of iterations. kAvgIterations = 1U << 3U, // Mark the counter as a iteration-average rate. See above. - kAvgIterationsRate = kIsRate | kAvgIterations + kAvgIterationsRate = kIsRate | kAvgIterations, + + // In the end, invert the result. This is always done last! + kInvert = 1U << 31U }; enum OneK { diff --git a/src/benchmark.cc b/src/benchmark.cc index 67c9744..9af0701 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -51,66 +51,60 @@ #include "thread_manager.h" #include "thread_timer.h" -DEFINE_bool(benchmark_list_tests, false, - "Print a list of benchmarks. This option overrides all other " - "options."); - -DEFINE_string(benchmark_filter, ".", - "A regular expression that specifies the set of benchmarks " - "to execute. If this flag is empty, or if this flag is the " - "string \"all\", all benchmarks linked into the binary are " - "run."); - -DEFINE_double(benchmark_min_time, 0.5, - "Minimum number of seconds we should run benchmark before " - "results are considered significant. For cpu-time based " - "tests, this is the lower bound on the total cpu time " - "used by all threads that make up the test. For real-time " - "based tests, this is the lower bound on the elapsed time " - "of the benchmark execution, regardless of number of " - "threads."); - -DEFINE_int32(benchmark_repetitions, 1, - "The number of runs of each benchmark. If greater than 1, the " - "mean and standard deviation of the runs will be reported."); - -DEFINE_bool( - benchmark_report_aggregates_only, false, - "Report the result of each benchmark repetitions. When 'true' is specified " - "only the mean, standard deviation, and other statistics are reported for " - "repeated benchmarks. Affects all reporters."); - -DEFINE_bool( - benchmark_display_aggregates_only, false, - "Display the result of each benchmark repetitions. When 'true' is " - "specified only the mean, standard deviation, and other statistics are " - "displayed for repeated benchmarks. Unlike " - "benchmark_report_aggregates_only, only affects the display reporter, but " - "*NOT* file reporter, which will still contain all the output."); - -DEFINE_string(benchmark_format, "console", - "The format to use for console output. Valid values are " - "'console', 'json', or 'csv'."); - -DEFINE_string(benchmark_out_format, "json", - "The format to use for file output. Valid values are " - "'console', 'json', or 'csv'."); - -DEFINE_string(benchmark_out, "", "The file to write additional output to"); - -DEFINE_string(benchmark_color, "auto", - "Whether to use colors in the output. Valid values: " - "'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use " - "colors if the output is being sent to a terminal and the TERM " - "environment variable is set to a terminal type that supports " - "colors."); - -DEFINE_bool(benchmark_counters_tabular, false, - "Whether to use tabular format when printing user counters to " - "the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0." - "Defaults to false."); - -DEFINE_int32(v, 0, "The level of verbose logging to output"); +// Print a list of benchmarks. This option overrides all other options. +DEFINE_bool(benchmark_list_tests, false); + +// A regular expression that specifies the set of benchmarks to execute. If +// this flag is empty, or if this flag is the string \"all\", all benchmarks +// linked into the binary are run. +DEFINE_string(benchmark_filter, "."); + +// Minimum number of seconds we should run benchmark before results are +// considered significant. For cpu-time based tests, this is the lower bound +// on the total cpu time used by all threads that make up the test. For +// real-time based tests, this is the lower bound on the elapsed time of the +// benchmark execution, regardless of number of threads. +DEFINE_double(benchmark_min_time, 0.5); + +// The number of runs of each benchmark. If greater than 1, the mean and +// standard deviation of the runs will be reported. +DEFINE_int32(benchmark_repetitions, 1); + +// Report the result of each benchmark repetitions. When 'true' is specified +// only the mean, standard deviation, and other statistics are reported for +// repeated benchmarks. Affects all reporters. +DEFINE_bool( benchmark_report_aggregates_only, false); + +// Display the result of each benchmark repetitions. When 'true' is specified +// only the mean, standard deviation, and other statistics are displayed for +// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects +// the display reporter, but *NOT* file reporter, which will still contain +// all the output. +DEFINE_bool( benchmark_display_aggregates_only, false); + +// The format to use for console output. +// Valid values are 'console', 'json', or 'csv'. +DEFINE_string(benchmark_format, "console"); + +// The format to use for file output. +// Valid values are 'console', 'json', or 'csv'. +DEFINE_string(benchmark_out_format, "json"); + +// The file to write additional output to. +DEFINE_string(benchmark_out, ""); + +// Whether to use colors in the output. Valid values: +// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if +// the output is being sent to a terminal and the TERM environment variable is +// set to a terminal type that supports colors. +DEFINE_string(benchmark_color, "auto"); + +// Whether to use tabular format when printing user counters to the console. +// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. +DEFINE_bool(benchmark_counters_tabular, false); + +// The level of verbose logging to output +DEFINE_int32(v, 0); namespace benchmark { diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 5eaea82..afe5238 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -15,12 +15,11 @@ #define DECLARE_string(name) extern std::string FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val, doc) bool FLAG(name) = (default_val) -#define DEFINE_int32(name, default_val, doc) int32_t FLAG(name) = (default_val) -#define DEFINE_int64(name, default_val, doc) int64_t FLAG(name) = (default_val) -#define DEFINE_double(name, default_val, doc) double FLAG(name) = (default_val) -#define DEFINE_string(name, default_val, doc) \ - std::string FLAG(name) = (default_val) +#define DEFINE_bool(name, default_val) bool FLAG(name) = (default_val) +#define DEFINE_int32(name, default_val) int32_t FLAG(name) = (default_val) +#define DEFINE_int64(name, default_val) int64_t FLAG(name) = (default_val) +#define DEFINE_double(name, default_val) double FLAG(name) = (default_val) +#define DEFINE_string(name, default_val) std::string FLAG(name) = (default_val) namespace benchmark { // Parses a bool/Int32/string from the environment variable diff --git a/src/console_reporter.cc b/src/console_reporter.cc index cc8ae27..6fd7645 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -12,21 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "benchmark/benchmark.h" -#include "complexity.h" -#include "counter.h" - #include <algorithm> #include <cstdint> #include <cstdio> +#include <cstring> #include <iostream> #include <string> #include <tuple> #include <vector> +#include "benchmark/benchmark.h" #include "check.h" #include "colorprint.h" #include "commandlineflags.h" +#include "complexity.h" +#include "counter.h" #include "internal_macros.h" #include "string_util.h" #include "timers.h" @@ -156,16 +156,14 @@ void ConsoleReporter::PrintRunData(const Run& result) { const std::size_t cNameLen = std::max(std::string::size_type(10), c.first.length()); auto const& s = HumanReadableNumber(c.second.value, c.second.oneK); + const char* unit = ""; + if (c.second.flags & Counter::kIsRate) + unit = (c.second.flags & Counter::kInvert) ? "s" : "/s"; if (output_options_ & OO_Tabular) { - if (c.second.flags & Counter::kIsRate) { - printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str()); - } else { - printer(Out, COLOR_DEFAULT, " %*s", cNameLen, s.c_str()); - } - } else { - const char* unit = (c.second.flags & Counter::kIsRate) ? "/s" : ""; - printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), + printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(), unit); + } else { + printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit); } } diff --git a/src/counter.cc b/src/counter.cc index c248ea1..cf5b78e 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -32,6 +32,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time, if (c.flags & Counter::kAvgIterations) { v /= iterations; } + + if (c.flags & Counter::kInvert) { // Invert is *always* last. + v = 1.0 / v; + } return v; } diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 4d308df..5699f4f 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -149,6 +149,89 @@ void CheckRate(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); // ========================================================================= // +// ----------------------- Inverted Counters Output ------------------------ // +// ========================================================================= // + +void BM_Invert(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; + state.counters["bar"] = bm::Counter{10000, bm::Counter::kInvert}; +} +BENCHMARK(BM_Invert); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, + {"\"run_name\": \"BM_Invert\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Invert\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckInvert(Results const& e) { + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 10000, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 0.0001, 0.0001); +} +CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); + +// ========================================================================= // +// ------------------------- InvertedRate Counters Output +// -------------------------- // +// ========================================================================= // + +void BM_Counters_InvertedRate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + } + namespace bm = benchmark; + state.counters["foo"] = + bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert}; + state.counters["bar"] = + bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert}; +} +BENCHMARK(BM_Counters_InvertedRate); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report " + "bar=%hrfloats foo=%hrfloats$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_InvertedRate\",$"}, + {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_InvertedRate\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckInvertedRate(Results const& e) { + double t = e.DurationCPUTime(); // this (and not real time) is the time used + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / 8192.0, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate", &CheckInvertedRate); + +// ========================================================================= // // ------------------------- Thread Counters Output ------------------------ // // ========================================================================= // |