From 9e47c31de6940a2b28d40d50ebb2e1fad6963c95 Mon Sep 17 00:00:00 2001 From: Zhizhou Yang Date: Wed, 24 Oct 2018 16:20:32 -0700 Subject: bisect tool: Adding unit tests for pass/transform level bisect This patch added unit tests for pass and transformation level bisect to the bisecting tool. Since no compiler is involved in unit test, it used cmd_script.py to simulate the output of compiler (to stderr). BUG=chromium:878954 TEST=Passed all unit tests. Change-Id: I599c079497333ec24f08f37c3315a16f28c0f887 Reviewed-on: https://chromium-review.googlesource.com/1298384 Commit-Ready: Zhizhou Yang Tested-by: Zhizhou Yang Reviewed-by: Caroline Tice --- .../test/binary_search_tool_tester.py | 152 ++++++++++++++++++--- binary_search_tool/test/cmd_script.py | 71 ++++++++++ binary_search_tool/test/cmd_script_no_support.py | 23 ++++ binary_search_tool/test/gen_obj.py | 2 +- binary_search_tool/test/generate_cmd.py | 25 ++++ 5 files changed, 253 insertions(+), 20 deletions(-) create mode 100755 binary_search_tool/test/cmd_script.py create mode 100644 binary_search_tool/test/cmd_script_no_support.py create mode 100755 binary_search_tool/test/generate_cmd.py (limited to 'binary_search_tool') diff --git a/binary_search_tool/test/binary_search_tool_tester.py b/binary_search_tool/test/binary_search_tool_tester.py index 923ea112..aff45a86 100755 --- a/binary_search_tool/test/binary_search_tool_tester.py +++ b/binary_search_tool/test/binary_search_tool_tester.py @@ -132,7 +132,7 @@ class BisectingUtilsTest(unittest.TestCase): cleanup_list = [ './is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad', - 'noinc_prune_good' + 'noinc_prune_good', './cmd_script.sh' ] for f in cleanup_list: if os.path.exists(f): @@ -304,24 +304,6 @@ class BisectingUtilsTest(unittest.TestCase): found_obj = int(bss.found_items.pop()) self.assertEquals(bad_objs[found_obj], 1) - def test_pass_bisect(self): - bss = binary_search_state.MockBinarySearchState( - get_initial_items='./gen_init_list.py', - switch_to_good='./switch_to_good.py', - switch_to_bad='./switch_to_bad.py', - pass_bisect='./generate_cmd.py', - test_script='./is_good.py', - test_setup_script='./test_setup.py', - prune=False, - file_args=True) - # TODO: Need to design unit tests for pass level bisection - bss.DoSearchBadItems() - self.assertEquals(len(bss.found_items), 1) - - bad_objs = common.ReadObjectsFile() - found_obj = int(bss.found_items.pop()) - self.assertEquals(bad_objs[found_obj], 1) - def test_set_file(self): binary_search_state.Run( get_initial_items='./gen_init_list.py', @@ -367,6 +349,131 @@ class BisectingUtilsTest(unittest.TestCase): self.assertEqual(actual_result, expected_result) +class BisectingUtilsPassTest(BisectingUtilsTest): + """Tests for bisecting tool at pass/transformation level.""" + + def check_pass_output(self, pass_name, pass_num, trans_num): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad pass: " logs/binary_search_tool_tester.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + _, _, bad_info = line.partition('Bad pass: ') + actual_info = pass_name + ' at number ' + str(pass_num) + self.assertEqual(actual_info, bad_info) + + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad transformation number: ' + '" logs/binary_search_tool_tester.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + _, _, bad_info = line.partition('Bad transformation number: ') + actual_info = str(trans_num) + self.assertEqual(actual_info, bad_info) + + def test_with_prune(self): + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=True, + file_args=True) + self.assertEquals(ret, 1) + + def test_gen_cmd_script(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + bss.DoSearchBadItems() + cmd_script_path = bss.cmd_script + self.assertTrue(os.path.exists(cmd_script_path)) + + def test_no_pass_support(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + bss.cmd_script = './cmd_script_no_support.py' + # No support for -opt-bisect-limit + with self.assertRaises(RuntimeError): + bss.BuildWithPassLimit(-1) + + def test_no_transform_support(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + bss.cmd_script = './cmd_script_no_support.py' + # No support for -print-debug-counter + with self.assertRaises(RuntimeError): + bss.BuildWithTransformLimit(-1, 'counter_name') + + def test_pass_transform_bisect(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + pass_num = 4 + trans_num = 19 + bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num) + bss.DoSearchBadPass() + self.check_pass_output('instcombine-visit', pass_num, trans_num) + + def test_result_not_reproduced_pass(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + # Fails reproducing at pass level. + pass_num = 0 + trans_num = 19 + bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num) + with self.assertRaises(ValueError): + bss.DoSearchBadPass() + + def test_result_not_reproduced_transform(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + # Fails reproducing at transformation level. + pass_num = 4 + trans_num = 0 + bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num) + with self.assertRaises(ValueError): + bss.DoSearchBadPass() + + class BisectStressTest(unittest.TestCase): """Stress tests for bisecting tool.""" @@ -442,6 +549,13 @@ def Main(argv): suite.addTest(BisectingUtilsTest('test_no_prune')) suite.addTest(BisectingUtilsTest('test_set_file')) suite.addTest(BisectingUtilsTest('test_noincremental_prune')) + suite.addTest(BisectingUtilsPassTest('test_with_prune')) + suite.addTest(BisectingUtilsPassTest('test_gen_cmd_script')) + suite.addTest(BisectingUtilsPassTest('test_no_pass_support')) + suite.addTest(BisectingUtilsPassTest('test_no_transform_support')) + suite.addTest(BisectingUtilsPassTest('test_pass_transform_bisect')) + suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_pass')) + suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_transform')) suite.addTest(BisectTest('test_full_bisector')) suite.addTest(BisectStressTest('test_every_obj_bad')) suite.addTest(BisectStressTest('test_every_index_is_bad')) diff --git a/binary_search_tool/test/cmd_script.py b/binary_search_tool/test/cmd_script.py new file mode 100755 index 00000000..6940eaae --- /dev/null +++ b/binary_search_tool/test/cmd_script.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python2 +"""Command script without compiler support for pass level bisection. + +This script generates a pseudo log which a workable compiler should print out. +It assumes that -opt-bisect-limit and -print-debug-counter are supported by the +compiler. +""" + +from __future__ import print_function + +import os +import sys + +import common + + +def Main(argv): + if not os.path.exists('./is_setup'): + return 1 + + if len(argv) != 3: + return 1 + + limit_flags = os.environ['LIMIT_FLAGS'] + opt_bisect_exist = False + debug_counter_exist = False + + for option in limit_flags.split(): + if '-opt-bisect-limit' in option: + opt_bisect_limit = int(option.split('=')[-1]) + opt_bisect_exist = True + if '-debug-counter=' in option: + debug_counter = int(option.split('=')[-1]) + debug_counter_exist = True + + if not opt_bisect_exist: + return 1 + + # Manually set total number and bad number + total_pass = 10 + total_transform = 20 + bad_pass = int(argv[1]) + bad_transform = int(argv[2]) + + if opt_bisect_limit == -1: + opt_bisect_limit = total_pass + + for i in xrange(1, total_pass + 1): + bisect_str = 'BISECT: %srunning pass (%d) Combine redundant ' \ + 'instructions on function (f1)' \ + % ('NOT ' if i > opt_bisect_limit else '', i) + print(bisect_str, file=sys.stderr) + + if debug_counter_exist: + print('Counters and values:', file=sys.stderr) + print( + 'instcombine-visit : {%d, 0, %d}' % (total_transform, debug_counter), + file=sys.stderr) + + if opt_bisect_limit > bad_pass or \ + (debug_counter_exist and debug_counter > bad_transform): + common.WriteWorkingSet([1]) + else: + common.WriteWorkingSet([0]) + + return 0 + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/binary_search_tool/test/cmd_script_no_support.py b/binary_search_tool/test/cmd_script_no_support.py new file mode 100644 index 00000000..a817f300 --- /dev/null +++ b/binary_search_tool/test/cmd_script_no_support.py @@ -0,0 +1,23 @@ +"""Command script without compiler support for pass level bisection. + +This script generates a pseudo log when certain bisecting flags are not +supported by compiler. +""" + +from __future__ import print_function + +import os +import sys + + +def Main(): + if not os.path.exists('./is_setup'): + return 1 + print('No support for -opt-bisect-limit or -print-debug-counter.', + file=sys.stderr) + return 0 + + +if __name__ == '__main__': + retval = Main() + sys.exit(retval) diff --git a/binary_search_tool/test/gen_obj.py b/binary_search_tool/test/gen_obj.py index d17e93f5..a2bc7d93 100755 --- a/binary_search_tool/test/gen_obj.py +++ b/binary_search_tool/test/gen_obj.py @@ -85,7 +85,7 @@ def Main(argv): f.close() obj_num = len(obj_list) - bad_obj_num = obj_list.count('1') + bad_obj_num = obj_list.count(1) print('Generated {0} object files, with {1} bad ones.'.format( obj_num, bad_obj_num)) diff --git a/binary_search_tool/test/generate_cmd.py b/binary_search_tool/test/generate_cmd.py new file mode 100755 index 00000000..f6876eda --- /dev/null +++ b/binary_search_tool/test/generate_cmd.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python2 +"""Generate a virtual cmd script for pass level bisection. + +This is a required argument for pass level bisecting. For unit test, we use +this script to verify if cmd_script.sh is generated correctly. +""" + +from __future__ import print_function + +import os +import sys + + +def Main(): + if not os.path.exists('./is_setup'): + return 1 + file_name = 'cmd_script.sh' + with open(file_name, 'w') as f: + f.write('Generated by generate_cmd.py') + return 0 + + +if __name__ == '__main__': + retval = Main() + sys.exit(retval) -- cgit v1.2.3