Properly record extra errors. (#284)

* Include exception objects in record for tests' procedure functions to consume
* Introduce a wrapper class for storing exception objects in record, including info like stacktrace.
* Introduce an alias for the main exception.
* Mark failed tests as fail despite subsequent errors..

1 files changed, 71 insertions, 26 deletions

diff --git a/tests/mobly/base_test_test.py b/tests/mobly/base_test_test.py
index 1cc1078..bd7dce9 100755
--- a/tests/mobly/base_test_test.py
+++ b/tests/mobly/base_test_test.py
@@ -250,7 +250,7 @@ class BaseTestTest(unittest.TestCase):
     def test_teardown_test_assert_fail(self):
         class MockBaseTest(base_test.BaseTestClass):
             def teardown_test(self):
-                asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
+                asserts.fail(MSG_EXPECTED_EXCEPTION)
 
             def test_something(self):
                 pass
@@ -259,7 +259,7 @@ class BaseTestTest(unittest.TestCase):
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertIsNone(actual_record.details)
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
@@ -277,10 +277,9 @@ class BaseTestTest(unittest.TestCase):
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertIsNone(actual_record.details)
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
-        expected_extra_error = {"teardown_test": MSG_EXPECTED_EXCEPTION}
-        self.assertEqual(actual_record.extra_errors, expected_extra_error)
+        self.assertFalse(actual_record.extra_errors)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
@@ -372,10 +371,10 @@ class BaseTestTest(unittest.TestCase):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
-        my_mock.assert_called_once_with("on_fail")
+        my_mock.assert_called_once_with('on_fail')
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertIsNone(actual_record.details)
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
@@ -463,8 +462,8 @@ class BaseTestTest(unittest.TestCase):
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
-        self.assertEqual(actual_record.extra_errors,
-                         {'teardown_test': 'This is an expected exception.ha'})
+        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
+                         'This is an expected exception.ha')
         self.assertIsNone(actual_record.extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
@@ -504,12 +503,12 @@ class BaseTestTest(unittest.TestCase):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
-        actual_record = bt_cls.results.error[0]
+        actual_record = bt_cls.results.failed[0]
         self.assertIn('_on_fail', actual_record.extra_errors)
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
-        expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
+        expected_summary = ("Error 0, Executed 1, Failed 1, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
 
@@ -526,8 +525,8 @@ class BaseTestTest(unittest.TestCase):
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
-        expected_extra_error = {'_on_pass': expected_msg}
-        self.assertEqual(actual_record.extra_errors, expected_extra_error)
+        self.assertEqual(actual_record.extra_errors['_on_pass'].details,
+                         expected_msg)
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
@@ -549,12 +548,58 @@ class BaseTestTest(unittest.TestCase):
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, "Test Body Exception.")
         self.assertIsNone(actual_record.extras)
-        self.assertEqual(actual_record.extra_errors["teardown_test"],
-                         "Details=This is an expected exception., Extras=None")
+        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
+                         MSG_EXPECTED_EXCEPTION)
+        self.assertIsNone(actual_record.extra_errors['teardown_test'].extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
 
+    def test_exception_objects_in_record(self):
+        """Checks that the exception objects are correctly tallied.
+        """
+        expected_termination_signal = Exception('Test Body Exception.')
+        expected_extra_error = Exception('teardown_test Exception.')
+
+        class MockBaseTest(base_test.BaseTestClass):
+            def teardown_test(self):
+                raise expected_extra_error
+
+            def test_something(self):
+                raise expected_termination_signal
+
+        bt_cls = MockBaseTest(self.mock_test_cls_configs)
+        bt_cls.run()
+        actual_record = bt_cls.results.error[0]
+        self.assertIs(actual_record.termination_signal.exception,
+                      expected_termination_signal)
+        self.assertIsNotNone(actual_record.termination_signal.stacktrace)
+        self.assertEqual(len(actual_record.extra_errors), 1)
+        extra_error = actual_record.extra_errors['teardown_test']
+        self.assertIs(extra_error.exception, expected_extra_error)
+        self.assertIsNotNone(extra_error.stacktrace)
+        self.assertIsNone(actual_record.extras)
+
+    def test_promote_extra_errors_to_termination_signal(self):
+        """If no termination singal is specified, use the first extra error as
+        the termination signal.
+        """
+        expected_extra_error = Exception('teardown_test Exception.')
+
+        class MockBaseTest(base_test.BaseTestClass):
+            def teardown_test(self):
+                raise expected_extra_error
+
+            def test_something(self):
+                pass
+
+        bt_cls = MockBaseTest(self.mock_test_cls_configs)
+        bt_cls.run()
+        actual_record = bt_cls.results.error[0]
+        self.assertFalse(actual_record.extra_errors)
+        self.assertEqual(actual_record.details, 'teardown_test Exception.')
+        self.assertIsNotNone(actual_record.stacktrace)
+
     def test_explicit_pass_but_teardown_test_raises_an_exception(self):
         """Test record result should be marked as ERROR as opposed to PASS.
         """
@@ -564,16 +609,17 @@ class BaseTestTest(unittest.TestCase):
                 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
 
             def test_something(self):
-                asserts.explicit_pass("Test Passed!")
+                asserts.explicit_pass('Test Passed!')
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertEqual(actual_record.details, "Test Passed!")
+        self.assertEqual(actual_record.details, 'Test Passed!')
         self.assertIsNone(actual_record.extras)
-        self.assertEqual(actual_record.extra_errors["teardown_test"],
-                         "Details=This is an expected exception., Extras=None")
+        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
+                         MSG_EXPECTED_EXCEPTION)
+        self.assertIsNone(actual_record.extra_errors['teardown_test'].extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
@@ -606,8 +652,8 @@ class BaseTestTest(unittest.TestCase):
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertEqual(actual_record.extras, MOCK_EXTRA)
-        self.assertEqual(actual_record.extra_errors,
-                         {'_on_pass': MSG_EXPECTED_EXCEPTION})
+        self.assertEqual(actual_record.extra_errors['_on_pass'].details,
+                         MSG_EXPECTED_EXCEPTION)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
@@ -622,14 +668,13 @@ class BaseTestTest(unittest.TestCase):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
-        actual_record = bt_cls.results.error[0]
-        self.assertEqual(bt_cls.results.failed, [])
+        actual_record = bt_cls.results.failed[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertEqual(actual_record.extras, MOCK_EXTRA)
-        self.assertEqual(actual_record.extra_errors,
-                         {'_on_fail': MSG_EXPECTED_EXCEPTION})
-        expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
+        self.assertEqual(actual_record.extra_errors['_on_fail'].details,
+                         MSG_EXPECTED_EXCEPTION)
+        expected_summary = ("Error 0, Executed 1, Failed 1, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)