aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHaibo Huang <hhb@google.com>2019-09-06 17:31:57 -0700
committerHaibo Huang <hhb@google.com>2019-09-06 17:31:57 -0700
commit64bb0ff47f741d206b9676cf69e1435cc50ad512 (patch)
tree75b34f607b8cbe846843582a101f502829beb072
parent984fd060925a9c1ba5a3944e9c116464771f84c1 (diff)
parent7ee72863fdb1ccb2af5a011250b56af3f49b7511 (diff)
downloadgoogle-benchmark-64bb0ff47f741d206b9676cf69e1435cc50ad512.tar.gz
Upgrade google-benchmark to 7ee72863fdb1ccb2af5a011250b56af3f49b7511ndk-sysroot-r21
Test: None Change-Id: I04175dd39de9d9768851642576873359befbac4e
-rw-r--r--.gitignore1
-rw-r--r--AUTHORS1
-rw-r--r--CONTRIBUTORS1
-rw-r--r--METADATA6
-rw-r--r--README.md30
-rw-r--r--include/benchmark/benchmark.h5
-rw-r--r--src/benchmark.cc114
-rw-r--r--src/commandlineflags.h11
-rw-r--r--src/console_reporter.cc22
-rw-r--r--src/counter.cc4
-rw-r--r--test/user_counters_test.cc83
11 files changed, 192 insertions, 86 deletions
diff --git a/.gitignore b/.gitignore
index 806d04c..a7716e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
!/cmake/*.cmake
!/test/AssemblyTests.cmake
*~
+*.swp
*.pyc
__pycache__
diff --git a/AUTHORS b/AUTHORS
index 1541863..c5e5c0c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -44,6 +44,7 @@ Ori Livneh <ori.livneh@gmail.com>
Paul Redmond <paul.redmond@gmail.com>
Radoslav Yovchev <radoslav.tm@gmail.com>
Roman Lebedev <lebedev.ri@gmail.com>
+Sayan Bhattacharjee <aero.sayan@gmail.com>
Shuo Chen <chenshuo@chenshuo.com>
Steinar H. Gunderson <sgunderson@bigfoot.com>
Stripe, Inc.
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 167165f..5be6dc4 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -65,6 +65,7 @@ Raul Marin <rmrodriguez@cartodb.com>
Ray Glover <ray.glover@uk.ibm.com>
Robert Guo <robert.guo@mongodb.com>
Roman Lebedev <lebedev.ri@gmail.com>
+Sayan Bhattacharjee <aero.sayan@gmail.com>
Shuo Chen <chenshuo@chenshuo.com>
Tobias Ulvgård <tobias.ulvgard@dirac.se>
Tom Madams <tom.ej.madams@gmail.com> <tmadams@google.com>
diff --git a/METADATA b/METADATA
index 238cf6d..92a90f4 100644
--- a/METADATA
+++ b/METADATA
@@ -9,10 +9,10 @@ third_party {
type: GIT
value: "https://github.com/google/benchmark.git"
}
- version: "c408461983dd3adf49d450d7db926fc46f1d99a0"
+ version: "7ee72863fdb1ccb2af5a011250b56af3f49b7511"
last_upgrade_date {
year: 2019
- month: 8
- day: 7
+ month: 9
+ day: 6
}
}
diff --git a/README.md b/README.md
index f576973..eb9374c 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@ The following minimum versions are required to build the library:
* GCC 4.8
* Clang 3.4
-* Visual Studio 2013
+* Visual Studio 14 2015
* Intel 2015 Update 1
## Installation
@@ -422,6 +422,22 @@ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10);
```
Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ].
+The preceding code shows a method of defining a sparse range. The following
+example shows a method of defining a dense range. It is then used to benchmark
+the performance of `std::vector` initialization for uniformly increasing sizes.
+
+```c++
+static void BM_DenseRange(benchmark::State& state) {
+ for(auto _ : state) {
+ std::vector<int> v(state.range(0), state.range(0));
+ benchmark::DoNotOptimize(v.data());
+ benchmark::ClobberMemory();
+ }
+}
+BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128);
+```
+Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ].
+
You might have a benchmark that depends on two or more inputs. For example, the
following code defines a family of benchmarks for measuring the speed of set
insertion.
@@ -662,9 +678,9 @@ the resulting sum is the value which will be shown for the benchmark.
The `Counter` constructor accepts three parameters: the value as a `double`
; a bit flag which allows you to show counters as rates, and/or as per-thread
-iteration, and/or as per-thread averages, and/or iteration invariants;
-and a flag specifying the 'unit' - i.e. is 1k a 1000 (default,
-`benchmark::Counter::OneK::kIs1000`), or 1024
+iteration, and/or as per-thread averages, and/or iteration invariants,
+and/or finally inverting the result; and a flag specifying the 'unit' - i.e.
+is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024
(`benchmark::Counter::OneK::kIs1024`)?
```c++
@@ -673,8 +689,14 @@ and a flag specifying the 'unit' - i.e. is 1k a 1000 (default,
// Set the counter as a rate. It will be presented divided
// by the duration of the benchmark.
+ // Meaning: per one second, how many 'foo's are processed?
state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate);
+ // Set the counter as a rate. It will be presented divided
+ // by the duration of the benchmark, and the result inverted.
+ // Meaning: how many seconds it takes to process one 'foo'?
+ state.counters["FooInvRate"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+
// Set the counter as a thread-average quantity. It will
// be presented divided by the number of threads.
state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads);
diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h
index 856605f..144e212 100644
--- a/include/benchmark/benchmark.h
+++ b/include/benchmark/benchmark.h
@@ -368,7 +368,10 @@ class Counter {
// It will be presented divided by the number of iterations.
kAvgIterations = 1U << 3U,
// Mark the counter as a iteration-average rate. See above.
- kAvgIterationsRate = kIsRate | kAvgIterations
+ kAvgIterationsRate = kIsRate | kAvgIterations,
+
+ // In the end, invert the result. This is always done last!
+ kInvert = 1U << 31U
};
enum OneK {
diff --git a/src/benchmark.cc b/src/benchmark.cc
index 67c9744..9af0701 100644
--- a/src/benchmark.cc
+++ b/src/benchmark.cc
@@ -51,66 +51,60 @@
#include "thread_manager.h"
#include "thread_timer.h"
-DEFINE_bool(benchmark_list_tests, false,
- "Print a list of benchmarks. This option overrides all other "
- "options.");
-
-DEFINE_string(benchmark_filter, ".",
- "A regular expression that specifies the set of benchmarks "
- "to execute. If this flag is empty, or if this flag is the "
- "string \"all\", all benchmarks linked into the binary are "
- "run.");
-
-DEFINE_double(benchmark_min_time, 0.5,
- "Minimum number of seconds we should run benchmark before "
- "results are considered significant. For cpu-time based "
- "tests, this is the lower bound on the total cpu time "
- "used by all threads that make up the test. For real-time "
- "based tests, this is the lower bound on the elapsed time "
- "of the benchmark execution, regardless of number of "
- "threads.");
-
-DEFINE_int32(benchmark_repetitions, 1,
- "The number of runs of each benchmark. If greater than 1, the "
- "mean and standard deviation of the runs will be reported.");
-
-DEFINE_bool(
- benchmark_report_aggregates_only, false,
- "Report the result of each benchmark repetitions. When 'true' is specified "
- "only the mean, standard deviation, and other statistics are reported for "
- "repeated benchmarks. Affects all reporters.");
-
-DEFINE_bool(
- benchmark_display_aggregates_only, false,
- "Display the result of each benchmark repetitions. When 'true' is "
- "specified only the mean, standard deviation, and other statistics are "
- "displayed for repeated benchmarks. Unlike "
- "benchmark_report_aggregates_only, only affects the display reporter, but "
- "*NOT* file reporter, which will still contain all the output.");
-
-DEFINE_string(benchmark_format, "console",
- "The format to use for console output. Valid values are "
- "'console', 'json', or 'csv'.");
-
-DEFINE_string(benchmark_out_format, "json",
- "The format to use for file output. Valid values are "
- "'console', 'json', or 'csv'.");
-
-DEFINE_string(benchmark_out, "", "The file to write additional output to");
-
-DEFINE_string(benchmark_color, "auto",
- "Whether to use colors in the output. Valid values: "
- "'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use "
- "colors if the output is being sent to a terminal and the TERM "
- "environment variable is set to a terminal type that supports "
- "colors.");
-
-DEFINE_bool(benchmark_counters_tabular, false,
- "Whether to use tabular format when printing user counters to "
- "the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0."
- "Defaults to false.");
-
-DEFINE_int32(v, 0, "The level of verbose logging to output");
+// Print a list of benchmarks. This option overrides all other options.
+DEFINE_bool(benchmark_list_tests, false);
+
+// A regular expression that specifies the set of benchmarks to execute. If
+// this flag is empty, or if this flag is the string \"all\", all benchmarks
+// linked into the binary are run.
+DEFINE_string(benchmark_filter, ".");
+
+// Minimum number of seconds we should run benchmark before results are
+// considered significant. For cpu-time based tests, this is the lower bound
+// on the total cpu time used by all threads that make up the test. For
+// real-time based tests, this is the lower bound on the elapsed time of the
+// benchmark execution, regardless of number of threads.
+DEFINE_double(benchmark_min_time, 0.5);
+
+// The number of runs of each benchmark. If greater than 1, the mean and
+// standard deviation of the runs will be reported.
+DEFINE_int32(benchmark_repetitions, 1);
+
+// Report the result of each benchmark repetitions. When 'true' is specified
+// only the mean, standard deviation, and other statistics are reported for
+// repeated benchmarks. Affects all reporters.
+DEFINE_bool( benchmark_report_aggregates_only, false);
+
+// Display the result of each benchmark repetitions. When 'true' is specified
+// only the mean, standard deviation, and other statistics are displayed for
+// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects
+// the display reporter, but *NOT* file reporter, which will still contain
+// all the output.
+DEFINE_bool( benchmark_display_aggregates_only, false);
+
+// The format to use for console output.
+// Valid values are 'console', 'json', or 'csv'.
+DEFINE_string(benchmark_format, "console");
+
+// The format to use for file output.
+// Valid values are 'console', 'json', or 'csv'.
+DEFINE_string(benchmark_out_format, "json");
+
+// The file to write additional output to.
+DEFINE_string(benchmark_out, "");
+
+// Whether to use colors in the output. Valid values:
+// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if
+// the output is being sent to a terminal and the TERM environment variable is
+// set to a terminal type that supports colors.
+DEFINE_string(benchmark_color, "auto");
+
+// Whether to use tabular format when printing user counters to the console.
+// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.
+DEFINE_bool(benchmark_counters_tabular, false);
+
+// The level of verbose logging to output
+DEFINE_int32(v, 0);
namespace benchmark {
diff --git a/src/commandlineflags.h b/src/commandlineflags.h
index 5eaea82..afe5238 100644
--- a/src/commandlineflags.h
+++ b/src/commandlineflags.h
@@ -15,12 +15,11 @@
#define DECLARE_string(name) extern std::string FLAG(name)
// Macros for defining flags.
-#define DEFINE_bool(name, default_val, doc) bool FLAG(name) = (default_val)
-#define DEFINE_int32(name, default_val, doc) int32_t FLAG(name) = (default_val)
-#define DEFINE_int64(name, default_val, doc) int64_t FLAG(name) = (default_val)
-#define DEFINE_double(name, default_val, doc) double FLAG(name) = (default_val)
-#define DEFINE_string(name, default_val, doc) \
- std::string FLAG(name) = (default_val)
+#define DEFINE_bool(name, default_val) bool FLAG(name) = (default_val)
+#define DEFINE_int32(name, default_val) int32_t FLAG(name) = (default_val)
+#define DEFINE_int64(name, default_val) int64_t FLAG(name) = (default_val)
+#define DEFINE_double(name, default_val) double FLAG(name) = (default_val)
+#define DEFINE_string(name, default_val) std::string FLAG(name) = (default_val)
namespace benchmark {
// Parses a bool/Int32/string from the environment variable
diff --git a/src/console_reporter.cc b/src/console_reporter.cc
index cc8ae27..6fd7645 100644
--- a/src/console_reporter.cc
+++ b/src/console_reporter.cc
@@ -12,21 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include "benchmark/benchmark.h"
-#include "complexity.h"
-#include "counter.h"
-
#include <algorithm>
#include <cstdint>
#include <cstdio>
+#include <cstring>
#include <iostream>
#include <string>
#include <tuple>
#include <vector>
+#include "benchmark/benchmark.h"
#include "check.h"
#include "colorprint.h"
#include "commandlineflags.h"
+#include "complexity.h"
+#include "counter.h"
#include "internal_macros.h"
#include "string_util.h"
#include "timers.h"
@@ -156,16 +156,14 @@ void ConsoleReporter::PrintRunData(const Run& result) {
const std::size_t cNameLen = std::max(std::string::size_type(10),
c.first.length());
auto const& s = HumanReadableNumber(c.second.value, c.second.oneK);
+ const char* unit = "";
+ if (c.second.flags & Counter::kIsRate)
+ unit = (c.second.flags & Counter::kInvert) ? "s" : "/s";
if (output_options_ & OO_Tabular) {
- if (c.second.flags & Counter::kIsRate) {
- printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str());
- } else {
- printer(Out, COLOR_DEFAULT, " %*s", cNameLen, s.c_str());
- }
- } else {
- const char* unit = (c.second.flags & Counter::kIsRate) ? "/s" : "";
- printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(),
+ printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(),
unit);
+ } else {
+ printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit);
}
}
diff --git a/src/counter.cc b/src/counter.cc
index c248ea1..cf5b78e 100644
--- a/src/counter.cc
+++ b/src/counter.cc
@@ -32,6 +32,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time,
if (c.flags & Counter::kAvgIterations) {
v /= iterations;
}
+
+ if (c.flags & Counter::kInvert) { // Invert is *always* last.
+ v = 1.0 / v;
+ }
return v;
}
diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc
index 4d308df..5699f4f 100644
--- a/test/user_counters_test.cc
+++ b/test/user_counters_test.cc
@@ -149,6 +149,89 @@ void CheckRate(Results const& e) {
CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate);
// ========================================================================= //
+// ----------------------- Inverted Counters Output ------------------------ //
+// ========================================================================= //
+
+void BM_Invert(benchmark::State& state) {
+ for (auto _ : state) {
+ // This test requires a non-zero CPU time to avoid divide-by-zero
+ benchmark::DoNotOptimize(state.iterations());
+ }
+ namespace bm = benchmark;
+ state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert};
+ state.counters["bar"] = bm::Counter{10000, bm::Counter::kInvert};
+}
+BENCHMARK(BM_Invert);
+ADD_CASES(TC_ConsoleOut,
+ {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"},
+ {"\"run_name\": \"BM_Invert\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"repetitions\": 0,$", MR_Next},
+ {"\"repetition_index\": 0,$", MR_Next},
+ {"\"threads\": 1,$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_CSVOut, {{"^\"BM_Invert\",%csv_report,%float,%float$"}});
+// VS2013 does not allow this function to be passed as a lambda argument
+// to CHECK_BENCHMARK_RESULTS()
+void CheckInvert(Results const& e) {
+ CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 10000, 0.0001);
+ CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 0.0001, 0.0001);
+}
+CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert);
+
+// ========================================================================= //
+// ------------------------- InvertedRate Counters Output
+// -------------------------- //
+// ========================================================================= //
+
+void BM_Counters_InvertedRate(benchmark::State& state) {
+ for (auto _ : state) {
+ // This test requires a non-zero CPU time to avoid divide-by-zero
+ benchmark::DoNotOptimize(state.iterations());
+ }
+ namespace bm = benchmark;
+ state.counters["foo"] =
+ bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert};
+ state.counters["bar"] =
+ bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert};
+}
+BENCHMARK(BM_Counters_InvertedRate);
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report "
+ "bar=%hrfloats foo=%hrfloats$"}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_InvertedRate\",$"},
+ {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"repetitions\": 0,$", MR_Next},
+ {"\"repetition_index\": 0,$", MR_Next},
+ {"\"threads\": 1,$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_CSVOut,
+ {{"^\"BM_Counters_InvertedRate\",%csv_report,%float,%float$"}});
+// VS2013 does not allow this function to be passed as a lambda argument
+// to CHECK_BENCHMARK_RESULTS()
+void CheckInvertedRate(Results const& e) {
+ double t = e.DurationCPUTime(); // this (and not real time) is the time used
+ // check that the values are within 0.1% of the expected values
+ CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, t, 0.001);
+ CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / 8192.0, 0.001);
+}
+CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate", &CheckInvertedRate);
+
+// ========================================================================= //
// ------------------------- Thread Counters Output ------------------------ //
// ========================================================================= //