From 63699380829c94f636e2bd27680e761a37768628 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 2 Mar 2023 09:07:39 -0800 Subject: record: emit explicit end when appending to summary (#872) Since the `yaml` summary file is appended during the process, a reader is able to handle any new record as soon as they have been written. But it may read incomplete `yaml` data, for example on Linux there is no guaranty that a single write to file will be available as one read. By explicitly emitting the end marker to the record, the reader may now be able to read the complete `yaml` data before processing it. --- mobly/records.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mobly/records.py b/mobly/records.py index 69f1f9d..b77817c 100644 --- a/mobly/records.py +++ b/mobly/records.py @@ -156,6 +156,7 @@ class TestSummaryWriter: yaml.safe_dump(new_content, f, explicit_start=True, + explicit_end=True, allow_unicode=True, indent=4) -- cgit v1.2.3 From 61706b7bfb759269cebcc44d4888b6016782d34b Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Tue, 21 Mar 2023 10:37:39 -0700 Subject: suite_runner improvements (#875) As of the test runner: * Parse only known arguments. * Handle -l, --list_tests option. * Handle -tb, --test_bed option. * Handle -v, --verbose option. Also, do not force test name to only contains one dot `.` by passing `maxsplit=1` to `test_name.split`. This allow for more complex generated test names. --- mobly/suite_runner.py | 94 +++++++++++++++++++++++++++++++--------- tests/mobly/suite_runner_test.py | 17 ++++++++ 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/mobly/suite_runner.py b/mobly/suite_runner.py index 72732b5..b2b2579 100644 --- a/mobly/suite_runner.py +++ b/mobly/suite_runner.py @@ -92,22 +92,38 @@ def _parse_cli_args(argv): Namespace containing the parsed args. """ parser = argparse.ArgumentParser(description='Mobly Suite Executable.') - parser.add_argument('-c', - '--config', + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-c', + '--config', + type=str, + metavar='', + help='Path to the test configuration file.') + group.add_argument( + '-l', + '--list_tests', + action='store_true', + help='Print the names of the tests defined in a script without ' + 'executing them.') + parser.add_argument('--tests', + '--test_case', + nargs='+', type=str, - required=True, - metavar='', - help='Path to the test configuration file.') - parser.add_argument( - '--tests', - '--test_case', - nargs='+', - type=str, - metavar='[ClassA[.test_a] ClassB[.test_b] ...]', - help='A list of test classes and optional tests to execute.') + metavar='[ClassA[.test_a] ClassB[.test_b] ...]', + help='A list of test classes and optional tests to execute.') + parser.add_argument('-tb', + '--test_bed', + nargs='+', + type=str, + metavar='[ ...]', + help='Specify which test beds to run tests on.') + + parser.add_argument('-v', + '--verbose', + action='store_true', + help='Set console logger level to DEBUG') if not argv: argv = sys.argv[1:] - return parser.parse_args(argv) + return parser.parse_known_args(argv)[0] def _find_suite_class(): @@ -132,6 +148,33 @@ def _find_suite_class(): return test_suites[0] +def _print_test_names(test_classes): + """Prints the names of all the tests in all test classes. + Args: + test_classes: classes, the test classes to print names from. + """ + for test_class in test_classes: + cls = test_class(config_parser.TestRunConfig()) + test_names = [] + try: + # Executes pre-setup procedures, this is required since it might + # generate test methods that we want to return as well. + cls._pre_run() + if cls.tests: + # Specified by run list in class. + test_names = list(cls.tests) + else: + # No test method specified by user, list all in test class. + test_names = cls.get_existing_test_names() + except Exception: + logging.exception('Failed to retrieve generated tests.') + finally: + cls._clean_up() + print('==========> %s <==========' % cls.TAG) + for name in test_names: + print(f"{cls.TAG}.{name}") + + def run_suite_class(argv=None): """Executes tests in the test suite. @@ -139,17 +182,22 @@ def run_suite_class(argv=None): argv: A list that is then parsed as CLI args. If None, defaults to sys.argv. """ cli_args = _parse_cli_args(argv) - test_configs = config_parser.load_test_config_file(cli_args.config) + suite_class = _find_suite_class() + if cli_args.list_tests: + _print_test_names([suite_class]) + sys.exit(0) + test_configs = config_parser.load_test_config_file(cli_args.config, + cli_args.test_bed) config_count = len(test_configs) if config_count != 1: logging.error('Expect exactly one test config, found %d', config_count) config = test_configs[0] runner = test_runner.TestRunner( log_dir=config.log_path, testbed_name=config.testbed_name) - suite_class = _find_suite_class() suite = suite_class(runner, config) + console_level = logging.DEBUG if cli_args.verbose else logging.INFO ok = False - with runner.mobly_logger(): + with runner.mobly_logger(console_level=console_level): try: suite.setup_suite(config.copy()) try: @@ -176,8 +224,6 @@ def run_suite(test_classes, argv=None): input. """ args = _parse_cli_args(argv) - # Load test config file. - test_configs = config_parser.load_test_config_file(args.config) # Check the classes that were passed in for test_class in test_classes: @@ -187,14 +233,22 @@ def run_suite(test_classes, argv=None): 'mobly.base_test.BaseTestClass', test_class) sys.exit(1) + if args.list_tests: + _print_test_names(test_classes) + sys.exit(0) + + # Load test config file. + test_configs = config_parser.load_test_config_file(args.config, + args.test_bed) # Find the full list of tests to execute selected_tests = compute_selected_tests(test_classes, args.tests) + console_level = logging.DEBUG if args.verbose else logging.INFO # Execute the suite ok = True for config in test_configs: runner = test_runner.TestRunner(config.log_path, config.testbed_name) - with runner.mobly_logger(): + with runner.mobly_logger(console_level=console_level): for (test_class, tests) in selected_tests.items(): runner.add_test_class(config, test_class, tests) try: @@ -260,7 +314,7 @@ def compute_selected_tests(test_classes, selected_tests): test_class_name_to_tests = collections.OrderedDict() for test_name in selected_tests: if '.' in test_name: # Has a test method - (test_class_name, test_name) = test_name.split('.') + (test_class_name, test_name) = test_name.split('.', maxsplit=1) if test_class_name not in test_class_name_to_tests: # Never seen this class before test_class_name_to_tests[test_class_name] = [test_name] diff --git a/tests/mobly/suite_runner_test.py b/tests/mobly/suite_runner_test.py index 976e7ef..dabf74f 100755 --- a/tests/mobly/suite_runner_test.py +++ b/tests/mobly/suite_runner_test.py @@ -156,6 +156,23 @@ class SuiteRunnerTest(unittest.TestCase): mock_called.teardown_suite.assert_called_once_with() mock_exit.assert_not_called() + def test_print_test_names(self): + mock_test_class = mock.MagicMock() + mock_cls_instance = mock.MagicMock() + mock_test_class.return_value = mock_cls_instance + suite_runner._print_test_names([mock_test_class]) + mock_cls_instance._pre_run.assert_called_once() + mock_cls_instance._clean_up.assert_called_once() + + def test_print_test_names_with_exception(self): + mock_test_class = mock.MagicMock() + mock_cls_instance = mock.MagicMock() + mock_test_class.return_value = mock_cls_instance + suite_runner._print_test_names([mock_test_class]) + mock_cls_instance._pre_run.side_effect = Exception( + 'Something went wrong.') + mock_cls_instance._clean_up.assert_called_once() + if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 384ebbe47fbc48af60d73b1bede1f78976612fc4 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 23 Mar 2023 11:17:25 -0700 Subject: Default `TestRunConfig.log_path` to `_DEFAULT_LOG_PATH` (#876) --- mobly/config_parser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mobly/config_parser.py b/mobly/config_parser.py index 0f31cdc..2f2da91 100644 --- a/mobly/config_parser.py +++ b/mobly/config_parser.py @@ -176,8 +176,7 @@ class TestRunConfig: """ def __init__(self): - # Init value is an empty string to avoid string joining errors. - self.log_path = '' + self.log_path = _DEFAULT_LOG_PATH # Deprecated, use 'testbed_name' self.test_bed_name = None self.testbed_name = None -- cgit v1.2.3