Add a way to terminate repeated runs. am: 94709289d9

Original change: https://android-review.googlesource.com/c/platform/system/testing/gtest_extras/+/1780485

Change-Id: Iff85e2f98f7bcfc3b38bcfaf218f95d0fa040845

5 files changed, 153 insertions, 6 deletions

diff --git a/Isolate.cpp b/Isolate.cpp
index 8f76f54..f7ea4e3 100644
--- a/Isolate.cpp
+++ b/Isolate.cpp
@@ -808,6 +808,10 @@ int Isolate::Run() {
 
     if (total_pass_tests_ + total_skipped_tests_ + total_xfail_tests_ != tests_.size()) {
       exit_code = 1;
+      if (options_.stop_on_error() && options_.num_iterations() > 1) {
+        printf("\nTerminating repeat run due to failing tests (iteration %d).\n", i + 1);
+        break;
+      }
     }
   }
 
diff --git a/Options.cpp b/Options.cpp
index 96842a1..7e54259 100644
--- a/Options.cpp
+++ b/Options.cpp
@@ -64,12 +64,12 @@ const std::unordered_map<std::string, Options::ArgInfo> Options::kArgs = {
      {FLAG_ENVIRONMENT_VARIABLE | FLAG_REQUIRES_VALUE | FLAG_CHILD, &Options::SetString}},
     {"gtest_death_test_style",
      {FLAG_ENVIRONMENT_VARIABLE | FLAG_REQUIRES_VALUE | FLAG_CHILD, nullptr}},
-    {"gtest_break_on_failure", {FLAG_ENVIRONMENT_VARIABLE | FLAG_INCOMPATIBLE, nullptr}},
+    {"gtest_break_on_failure", {FLAG_ENVIRONMENT_VARIABLE, &Options::SetBool}},
     {"gtest_catch_exceptions", {FLAG_ENVIRONMENT_VARIABLE | FLAG_INCOMPATIBLE, nullptr}},
     {"gtest_random_seed", {FLAG_ENVIRONMENT_VARIABLE | FLAG_INCOMPATIBLE, nullptr}},
     {"gtest_shuffle", {FLAG_ENVIRONMENT_VARIABLE | FLAG_INCOMPATIBLE, nullptr}},
     {"gtest_stream_result_to", {FLAG_ENVIRONMENT_VARIABLE | FLAG_INCOMPATIBLE, nullptr}},
-    {"gtest_throw_on_failure", {FLAG_ENVIRONMENT_VARIABLE | FLAG_INCOMPATIBLE, nullptr}},
+    {"gtest_throw_on_failure", {FLAG_ENVIRONMENT_VARIABLE, &Options::SetBool}},
     {"gtest_shard_index",
      {FLAG_ENVIRONMENT_VARIABLE | FLAG_REQUIRES_VALUE, &Options::SetNumericEnvOnly}},
     {"gtest_total_shards",
@@ -311,6 +311,7 @@ bool Options::Process(const std::vector<const char*>& args, std::vector<char*>*
   // Initialize the variables.
   job_count_ = static_cast<size_t>(sysconf(_SC_NPROCESSORS_ONLN));
   num_iterations_ = ::testing::GTEST_FLAG(repeat);
+  stop_on_error_ = false;
   numerics_.clear();
   numerics_["deadline_threshold_ms"] = kDefaultDeadlineThresholdMs;
   numerics_["slow_threshold_ms"] = kDefaultSlowThresholdMs;
@@ -325,6 +326,8 @@ bool Options::Process(const std::vector<const char*>& args, std::vector<char*>*
   bools_["gtest_print_time"] = ::testing::GTEST_FLAG(print_time);
   bools_["gtest_also_run_disabled_tests"] = ::testing::GTEST_FLAG(also_run_disabled_tests);
   bools_["gtest_list_tests"] = false;
+  bools_["gtest_break_on_failure"] = false;
+  bools_["gtest_throw_on_failure"] = false;
 
   // This does nothing, only added so that passing this option does not exit.
   bools_["gtest_format"] = true;
@@ -373,6 +376,10 @@ bool Options::Process(const std::vector<const char*>& args, std::vector<char*>*
     }
   }
 
+  if (bools_["gtest_break_on_failure"] || bools_["gtest_throw_on_failure"]) {
+    stop_on_error_ = true;
+  }
+
   return true;
 }
 
diff --git a/Options.h b/Options.h
index 883940a..ad51e18 100644
--- a/Options.h
+++ b/Options.h
@@ -36,6 +36,7 @@ class Options {
 
   size_t job_count() const { return job_count_; }
   int num_iterations() const { return num_iterations_; }
+  bool stop_on_error() const { return stop_on_error_; }
 
   uint64_t deadline_threshold_ms() const { return numerics_.at("deadline_threshold_ms"); }
   uint64_t slow_threshold_ms() const { return numerics_.at("slow_threshold_ms"); }
@@ -54,6 +55,7 @@ class Options {
  private:
   size_t job_count_;
   int num_iterations_;
+  bool stop_on_error_;
 
   std::unordered_map<std::string, bool> bools_;
   std::unordered_map<std::string, std::string> strings_;
diff --git a/tests/OptionsTest.cpp b/tests/OptionsTest.cpp
index 97089a3..434c4df 100644
--- a/tests/OptionsTest.cpp
+++ b/tests/OptionsTest.cpp
@@ -95,6 +95,7 @@ TEST_F(OptionsTest, check_defaults) {
   EXPECT_EQ("", options.xml_file());
   EXPECT_EQ("", options.filter());
   EXPECT_EQ(1, options.num_iterations());
+  EXPECT_FALSE(options.stop_on_error());
   EXPECT_TRUE(options.print_time());
   EXPECT_FALSE(options.allow_disabled_tests());
   EXPECT_FALSE(options.list_tests());
@@ -628,6 +629,27 @@ TEST_F(OptionsTest, gtest_flagfile_does_not_exist) {
   EXPECT_EQ("Unable to read data from file /this/does/not/exist\n", capture.str());
 }
 
+TEST_F(OptionsTest, stop_on_error) {
+  std::vector<const char*> cur_args{"ignore", "--gtest_break_on_failure"};
+  Options options;
+  ASSERT_TRUE(options.Process(cur_args, &child_args_));
+  EXPECT_TRUE(options.stop_on_error());
+  EXPECT_THAT(child_args_, ElementsAre(StrEq("ignore")));
+
+  ClearChildArgs();
+  cur_args = std::vector<const char*>{"ignore", "--gtest_throw_on_failure"};
+  ASSERT_TRUE(options.Process(cur_args, &child_args_));
+  EXPECT_TRUE(options.stop_on_error());
+  EXPECT_THAT(child_args_, ElementsAre(StrEq("ignore")));
+
+  ClearChildArgs();
+  cur_args =
+      std::vector<const char*>{"ignore", "--gtest_break_on_failure", "--gtest_throw_on_failure"};
+  ASSERT_TRUE(options.Process(cur_args, &child_args_));
+  EXPECT_TRUE(options.stop_on_error());
+  EXPECT_THAT(child_args_, ElementsAre(StrEq("ignore")));
+}
+
 void OptionsTest::CheckIncompatible(const std::string arg) {
   CapturedStdout capture;
   std::vector<const char*> cur_args{"ignore", arg.c_str()};
@@ -639,12 +661,10 @@ void OptionsTest::CheckIncompatible(const std::string arg) {
 }
 
 TEST_F(OptionsTest, incompatible) {
-  ASSERT_NO_FATAL_FAILURE(CheckIncompatible("--gtest_break_on_failure"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatible("--gtest_catch_exceptions"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatible("--gtest_random_seed"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatible("--gtest_shuffle"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatible("--gtest_stream_result_to"));
-  ASSERT_NO_FATAL_FAILURE(CheckIncompatible("--gtest_throw_on_failure"));
 }
 
 TEST_F(OptionsTest, verify_non_env_variables) {
@@ -663,6 +683,7 @@ TEST_F(OptionsTest, verify_non_env_variables) {
   EXPECT_EQ("", options.xml_file());
   EXPECT_EQ("", options.filter());
   EXPECT_EQ(1, options.num_iterations());
+  EXPECT_FALSE(options.stop_on_error());
   EXPECT_TRUE(options.print_time());
   EXPECT_FALSE(options.allow_disabled_tests());
   EXPECT_FALSE(options.list_tests());
@@ -901,6 +922,38 @@ TEST_F(OptionsTest, gtest_death_test_style_error_no_value_from_env) {
   ASSERT_NE(-1, unsetenv("GTEST_DEATH_TEST_STYLE"));
 }
 
+TEST_F(OptionsTest, stop_on_error_from_env) {
+  ASSERT_NE(-1, setenv("GTEST_BREAK_ON_FAILURE", "", 1));
+
+  std::vector<const char*> cur_args{"ignore"};
+  Options options;
+  ASSERT_TRUE(options.Process(cur_args, &child_args_));
+  EXPECT_TRUE(options.stop_on_error());
+  EXPECT_THAT(child_args_, ElementsAre(StrEq("ignore")));
+
+  ASSERT_NE(-1, unsetenv("GTEST_BREAK_ON_FAILURE"));
+
+  ASSERT_NE(-1, setenv("GTEST_THROW_ON_FAILURE", "", 1));
+
+  ClearChildArgs();
+  ASSERT_TRUE(options.Process(cur_args, &child_args_));
+  EXPECT_TRUE(options.stop_on_error());
+  EXPECT_THAT(child_args_, ElementsAre(StrEq("ignore")));
+
+  ASSERT_NE(-1, unsetenv("GTEST_THROW_ON_FAILURE"));
+
+  ASSERT_NE(-1, setenv("GTEST_BREAK_ON_FAILURE", "", 1));
+  ASSERT_NE(-1, setenv("GTEST_THROW_ON_FAILURE", "", 1));
+
+  ClearChildArgs();
+  ASSERT_TRUE(options.Process(cur_args, &child_args_));
+  EXPECT_TRUE(options.stop_on_error());
+  EXPECT_THAT(child_args_, ElementsAre(StrEq("ignore")));
+
+  ASSERT_NE(-1, unsetenv("GTEST_BREAK_ON_FAILURE"));
+  ASSERT_NE(-1, unsetenv("GTEST_THROW_ON_FAILURE"));
+}
+
 void OptionsTest::CheckIncompatibleFromEnv(const std::string env_var) {
   ASSERT_NE(-1, setenv(env_var.c_str(), "", 1));
 
@@ -926,12 +979,10 @@ void OptionsTest::CheckIncompatibleFromEnv(const std::string env_var) {
 }
 
 TEST_F(OptionsTest, incompatible_from_env) {
-  ASSERT_NO_FATAL_FAILURE(CheckIncompatibleFromEnv("GTEST_BREAK_ON_FAILURE"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatibleFromEnv("GTEST_CATCH_EXCEPTIONS"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatibleFromEnv("GTEST_RANDOM_SEED"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatibleFromEnv("GTEST_SHUFFLE"));
   ASSERT_NO_FATAL_FAILURE(CheckIncompatibleFromEnv("GTEST_STREAM_RESULT_TO"));
-  ASSERT_NO_FATAL_FAILURE(CheckIncompatibleFromEnv("GTEST_THROW_ON_FAILURE"));
 }
 
 }  // namespace gtest_extras
diff --git a/tests/SystemTests.cpp b/tests/SystemTests.cpp
index 821c055..2b434a9 100644
--- a/tests/SystemTests.cpp
+++ b/tests/SystemTests.cpp
@@ -1328,6 +1328,89 @@ TEST_F(SystemTests, verify_gtest_flagfile) {
       Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{flagfile.c_str()}));
 }
 
+TEST_F(SystemTests, verify_repeat_stop_on_error) {
+  std::string expected =
+      "Note: Google Test filter = *.DISABLED_fail\n"
+      "[==========] Running 1 test from 1 test suite (20 jobs).\n"
+      "[ RUN      ] SystemTests.DISABLED_fail\n"
+      "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
+      "Expected equality of these values:\n"
+      "  1\n"
+      "  0\n"
+      "SystemTests.DISABLED_fail exited with exitcode 1.\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail (XX ms)\n"
+      "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
+      "[  PASSED  ] 0 tests.\n"
+      "[  FAILED  ] 1 test, listed below:\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail\n"
+      "\n"
+      " 1 FAILED TEST\n"
+      "\n"
+      "Terminating repeat run due to failing tests (iteration 1).\n";
+  ASSERT_NO_FATAL_FAILURE(
+      Verify("*.DISABLED_fail", expected, 1,
+             std::vector<const char*>{"--gtest_repeat=2", "--gtest_break_on_failure"}));
+}
+
+TEST_F(SystemTests, verify_repeat_no_stop_on_error) {
+  std::string expected =
+      "Note: Google Test filter = *.DISABLED_fail\n"
+      "[==========] Running 1 test from 1 test suite (20 jobs).\n"
+      "[ RUN      ] SystemTests.DISABLED_fail\n"
+      "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
+      "Expected equality of these values:\n"
+      "  1\n"
+      "  0\n"
+      "SystemTests.DISABLED_fail exited with exitcode 1.\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail (XX ms)\n"
+      "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
+      "[  PASSED  ] 0 tests.\n"
+      "[  FAILED  ] 1 test, listed below:\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail\n"
+      "\n"
+      " 1 FAILED TEST\n"
+      "\n"
+      "Repeating all tests (iteration 2) . . .\n"
+      "\n"
+      "[==========] Running 1 test from 1 test suite (20 jobs).\n"
+      "[ RUN      ] SystemTests.DISABLED_fail\n"
+      "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
+      "Expected equality of these values:\n"
+      "  1\n"
+      "  0\n"
+      "SystemTests.DISABLED_fail exited with exitcode 1.\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail (XX ms)\n"
+      "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
+      "[  PASSED  ] 0 tests.\n"
+      "[  FAILED  ] 1 test, listed below:\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail\n"
+      "\n"
+      " 1 FAILED TEST\n";
+  ASSERT_NO_FATAL_FAILURE(
+      Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_repeat=2"}));
+}
+
+TEST_F(SystemTests, verify_single_no_terminate_message) {
+  std::string expected =
+      "Note: Google Test filter = *.DISABLED_fail\n"
+      "[==========] Running 1 test from 1 test suite (20 jobs).\n"
+      "[ RUN      ] SystemTests.DISABLED_fail\n"
+      "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
+      "Expected equality of these values:\n"
+      "  1\n"
+      "  0\n"
+      "SystemTests.DISABLED_fail exited with exitcode 1.\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail (XX ms)\n"
+      "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
+      "[  PASSED  ] 0 tests.\n"
+      "[  FAILED  ] 1 test, listed below:\n"
+      "[  FAILED  ] SystemTests.DISABLED_fail\n"
+      "\n"
+      " 1 FAILED TEST\n";
+  ASSERT_NO_FATAL_FAILURE(
+      Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_break_on_failure"}));
+}
+
 // These tests are used by the verify_disabled tests.
 TEST_F(SystemTests, always_pass) {}