Cert: Add error catching during setup and teardown of test

* Catch RpcError during starting and stopping of BT stack
* Catch Environment error when reading from backing log file

Bug: 197161864
Tag: #gd-refactor
Test: gd/cert/run
Change-Id: Iaa730fd21dd596887799c1db93ec0d45166a5461

4 files changed, 68 insertions, 57 deletions

diff --git a/blueberry/tests/gd/cert/gd_base_test.py b/blueberry/tests/gd/cert/gd_base_test.py
index 31907743a..37e2d2e3b 100644
--- a/blueberry/tests/gd/cert/gd_base_test.py
+++ b/blueberry/tests/gd/cert/gd_base_test.py
@@ -23,11 +23,11 @@ from grpc import RpcError
 
 from cert.gd_base_test_lib import setup_rootcanal
 from cert.gd_base_test_lib import teardown_rootcanal
-from cert.gd_base_test_lib import setup_test_core
-from cert.gd_base_test_lib import teardown_test_core
 from cert.gd_base_test_lib import dump_crashes_core
 from cert.gd_device_lib import generate_coverage_report_for_host
 
+from facade import rootservice_pb2 as facade_rootservice
+
 from blueberry.tests.gd.cert.context import get_current_context
 from blueberry.tests.gd.cert.gd_device import MOBLY_CONTROLLER_CONFIG_NAME as CONTROLLER_CONFIG_NAME
 from blueberry.tests.gd.cert.tracelogger import TraceLogger
@@ -109,10 +109,30 @@ class GdBaseTestClass(base_test.BaseTestClass):
                         self.cert_coverage_info, new_cert_coverage_info))
             self.cert_coverage_info = new_cert_coverage_info
 
-        setup_test_core(dut=self.dut, cert=self.cert, dut_module=self.dut_module, cert_module=self.cert_module)
+        try:
+            self.dut.rootservice.StartStack(
+                facade_rootservice.StartStackRequest(
+                    module_under_test=facade_rootservice.BluetoothModule.Value(self.dut_module)))
+        except RpcError as rpc_error:
+            asserts.fail("Failed to start DUT stack, RpcError={!r}".format(rpc_error))
+        try:
+            self.cert.rootservice.StartStack(
+                facade_rootservice.StartStackRequest(
+                    module_under_test=facade_rootservice.BluetoothModule.Value(self.cert_module)))
+        except RpcError as rpc_error:
+            asserts.fail("Failed to start CERT stack, RpcError={!r}".format(rpc_error))
+        self.dut.wait_channel_ready()
+        self.cert.wait_channel_ready()
 
     def teardown_test(self):
-        teardown_test_core(cert=self.cert, dut=self.dut)
+        try:
+            self.cert.rootservice.StopStack(facade_rootservice.StopStackRequest())
+        except RpcError as rpc_error:
+            asserts.fail("Failed to stop CERT stack, RpcError={!r}".format(rpc_error))
+        try:
+            self.dut.rootservice.StopStack(facade_rootservice.StopStackRequest())
+        except RpcError as rpc_error:
+            asserts.fail("Failed to stop DUT stack, RpcError={!r}".format(rpc_error))
         # Destroy GD device objects
         self._controller_manager.unregister_controllers()
         teardown_rootcanal(
diff --git a/gd/cert/gd_base_test.py b/gd/cert/gd_base_test.py
index 6362d04e9..a71fb7761 100644
--- a/gd/cert/gd_base_test.py
+++ b/gd/cert/gd_base_test.py
@@ -15,10 +15,6 @@
 #   limitations under the License.
 
 import importlib
-import logging
-import os
-import signal
-import subprocess
 import traceback
 
 from functools import wraps
@@ -28,21 +24,14 @@ from acts import asserts, signals
 from acts.context import get_current_context
 from acts.base_test import BaseTestClass
 
-from cert.async_subprocess_logger import AsyncSubprocessLogger
-from cert.os_utils import get_gd_root
-from cert.os_utils import read_crash_snippet_and_log_tail
-from cert.os_utils import is_subprocess_alive
-from cert.os_utils import make_ports_available
-from cert.os_utils import TerminalColor
 from cert.gd_device import MOBLY_CONTROLLER_CONFIG_NAME as CONTROLLER_CONFIG_NAME
-from facade import rootservice_pb2 as facade_rootservice
 from cert.gd_base_test_lib import setup_rootcanal
 from cert.gd_base_test_lib import teardown_rootcanal
-from cert.gd_base_test_lib import setup_test_core
-from cert.gd_base_test_lib import teardown_test_core
 from cert.gd_base_test_lib import dump_crashes_core
 from cert.gd_device_lib import generate_coverage_report_for_host
 
+from facade import rootservice_pb2 as facade_rootservice
+
 
 class GdBaseTestClass(BaseTestClass):
 
@@ -116,10 +105,30 @@ class GdBaseTestClass(BaseTestClass):
                         self.cert_coverage_info, new_cert_coverage_info))
             self.cert_coverage_info = new_cert_coverage_info
 
-        setup_test_core(dut=self.dut, cert=self.cert, dut_module=self.dut_module, cert_module=self.cert_module)
+        try:
+            self.dut.rootservice.StartStack(
+                facade_rootservice.StartStackRequest(
+                    module_under_test=facade_rootservice.BluetoothModule.Value(self.dut_module)))
+        except RpcError as rpc_error:
+            asserts.fail("Failed to start DUT stack, RpcError={!r}".format(rpc_error))
+        try:
+            self.cert.rootservice.StartStack(
+                facade_rootservice.StartStackRequest(
+                    module_under_test=facade_rootservice.BluetoothModule.Value(self.cert_module)))
+        except RpcError as rpc_error:
+            asserts.fail("Failed to start CERT stack, RpcError={!r}".format(rpc_error))
+        self.dut.wait_channel_ready()
+        self.cert.wait_channel_ready()
 
     def teardown_test(self):
-        teardown_test_core(cert=self.cert, dut=self.dut)
+        try:
+            self.cert.rootservice.StopStack(facade_rootservice.StopStackRequest())
+        except RpcError as rpc_error:
+            asserts.fail("Failed to stop CERT stack, RpcError={!r}".format(rpc_error))
+        try:
+            self.dut.rootservice.StopStack(facade_rootservice.StopStackRequest())
+        except RpcError as rpc_error:
+            asserts.fail("Failed to stop DUT stack, RpcError={!r}".format(rpc_error))
         # Destroy GD device objects
         self._controller_manager.unregister_controllers()
         teardown_rootcanal(
diff --git a/gd/cert/gd_base_test_lib.py b/gd/cert/gd_base_test_lib.py
index d97464d71..28da36e0a 100644
--- a/gd/cert/gd_base_test_lib.py
+++ b/gd/cert/gd_base_test_lib.py
@@ -14,15 +14,10 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-import importlib
 import logging
 import os
 import signal
 import subprocess
-import traceback
-
-from functools import wraps
-from grpc import RpcError
 
 from cert.async_subprocess_logger import AsyncSubprocessLogger
 from cert.os_utils import get_gd_root
@@ -30,8 +25,6 @@ from cert.os_utils import read_crash_snippet_and_log_tail
 from cert.os_utils import is_subprocess_alive
 from cert.os_utils import make_ports_available
 from cert.os_utils import TerminalColor
-from cert.gd_device import MOBLY_CONTROLLER_CONFIG_NAME as CONTROLLER_CONFIG_NAME
-from facade import rootservice_pb2 as facade_rootservice
 
 
 def setup_rootcanal(dut_module, cert_module, verbose_mode, log_path_base, controller_configs):
@@ -124,21 +117,6 @@ def teardown_rootcanal(rootcanal_running, rootcanal_process, rootcanal_logger, s
         rootcanal_logger.stop()
 
 
-def setup_test_core(dut, cert, dut_module, cert_module):
-    dut.rootservice.StartStack(
-        facade_rootservice.StartStackRequest(module_under_test=facade_rootservice.BluetoothModule.Value(dut_module),))
-    cert.rootservice.StartStack(
-        facade_rootservice.StartStackRequest(module_under_test=facade_rootservice.BluetoothModule.Value(cert_module),))
-
-    dut.wait_channel_ready()
-    cert.wait_channel_ready()
-
-
-def teardown_test_core(cert, dut):
-    cert.rootservice.StopStack(facade_rootservice.StopStackRequest())
-    dut.rootservice.StopStack(facade_rootservice.StopStackRequest())
-
-
 def dump_crashes_core(dut, cert, rootcanal_running, rootcanal_process, rootcanal_logpath):
     dut_crash, dut_log_tail = dut.get_crash_snippet_and_log_tail()
     cert_crash, cert_log_tail = cert.get_crash_snippet_and_log_tail()
diff --git a/gd/cert/os_utils.py b/gd/cert/os_utils.py
index fbb4e2395..b49f3ed6c 100644
--- a/gd/cert/os_utils.py
+++ b/gd/cert/os_utils.py
@@ -119,22 +119,26 @@ def read_crash_snippet_and_log_tail(logpath):
     asan = False
     asan_lines = []
 
-    with open(logpath) as f:
-        for _, line in enumerate(f):
-            last_20_lines.append(line)
-            asan_match = ASAN_OUTPUT_START_REGEX.match(line)
-            if asan or asan_match:
-                asan_lines.append(line)
-                asan = True
-                continue
-
-            host_crash_match = HOST_CRASH_LINE_REGEX.match(line)
-            if host_crash_match:
-                crash_line = host_crash_match.group("line").replace(gd_root_prefix, "")
-                if HOST_ABORT_HEADER in crash_line \
-                        and len(last_20_lines) > 1:
-                    abort_line = last_20_lines[-2]
-                crash_log_lines.append(crash_line)
+    try:
+        with open(logpath) as f:
+            for _, line in enumerate(f):
+                last_20_lines.append(line)
+                asan_match = ASAN_OUTPUT_START_REGEX.match(line)
+                if asan or asan_match:
+                    asan_lines.append(line)
+                    asan = True
+                    continue
+
+                host_crash_match = HOST_CRASH_LINE_REGEX.match(line)
+                if host_crash_match:
+                    crash_line = host_crash_match.group("line").replace(gd_root_prefix, "")
+                    if HOST_ABORT_HEADER in crash_line \
+                            and len(last_20_lines) > 1:
+                        abort_line = last_20_lines[-2]
+                    crash_log_lines.append(crash_line)
+    except EnvironmentError:
+        logging.error("Cannot open backing log file at {}".format(logpath))
+        return None, None
 
     log_tail_20 = "".join(last_20_lines)
     crash_snippet = ""