diff options
Diffstat (limited to 'binary_search_tool/test/binary_search_tool_test.py')
| -rwxr-xr-x | binary_search_tool/test/binary_search_tool_test.py | 569 |
1 files changed, 569 insertions, 0 deletions
diff --git a/binary_search_tool/test/binary_search_tool_test.py b/binary_search_tool/test/binary_search_tool_test.py new file mode 100755 index 00000000..ca9313b5 --- /dev/null +++ b/binary_search_tool/test/binary_search_tool_test.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for bisecting tool.""" + +from __future__ import division +from __future__ import print_function + +__author__ = 'shenhan@google.com (Han Shen)' + +import os +import random +import sys +import unittest + +from cros_utils import command_executer +from binary_search_tool import binary_search_state +from binary_search_tool import run_bisect + +from binary_search_tool.test import common +from binary_search_tool.test import gen_obj + + +def GenObj(): + obj_num = random.randint(100, 1000) + bad_obj_num = random.randint(obj_num // 100, obj_num // 20) + if bad_obj_num == 0: + bad_obj_num = 1 + gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)]) + + +def CleanObj(): + os.remove(common.OBJECTS_FILE) + os.remove(common.WORKING_SET_FILE) + print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE, + common.WORKING_SET_FILE)) + + +class BisectTest(unittest.TestCase): + """Tests for run_bisect.py""" + + def setUp(self): + with open('./is_setup', 'w', encoding='utf-8'): + pass + + try: + os.remove(binary_search_state.STATE_FILE) + except OSError: + pass + + def tearDown(self): + try: + os.remove('./is_setup') + os.remove(os.readlink(binary_search_state.STATE_FILE)) + os.remove(binary_search_state.STATE_FILE) + except OSError: + pass + + class FullBisector(run_bisect.Bisector): + """Test bisector to test run_bisect.py with""" + + def __init__(self, options, overrides): + super(BisectTest.FullBisector, self).__init__(options, overrides) + + def PreRun(self): + GenObj() + return 0 + + def Run(self): + return binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True) + + def PostRun(self): + CleanObj() + return 0 + + def test_full_bisector(self): + ret = run_bisect.Run(self.FullBisector({}, {})) + self.assertEqual(ret, 0) + self.assertFalse(os.path.exists(common.OBJECTS_FILE)) + self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) + + def check_output(self): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad items are: " logs/binary_search_tool_test.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + + _, _, bad_ones = line.partition('Bad items are: ') + bad_ones = bad_ones.split() + expected_result = common.ReadObjectsFile() + + # Reconstruct objects file from bad_ones and compare + actual_result = [0] * len(expected_result) + for bad_obj in bad_ones: + actual_result[int(bad_obj)] = 1 + + self.assertEqual(actual_result, expected_result) + + +class BisectingUtilsTest(unittest.TestCase): + """Tests for bisecting tool.""" + + def setUp(self): + """Generate [100-1000] object files, and 1-5% of which are bad ones.""" + GenObj() + + with open('./is_setup', 'w', encoding='utf-8'): + pass + + try: + os.remove(binary_search_state.STATE_FILE) + except OSError: + pass + + def tearDown(self): + """Cleanup temp files.""" + CleanObj() + + try: + os.remove(os.readlink(binary_search_state.STATE_FILE)) + except OSError: + pass + + cleanup_list = [ + './is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad', + 'noinc_prune_good', './cmd_script.sh' + ] + for f in cleanup_list: + if os.path.exists(f): + os.remove(f) + + def runTest(self): + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True) + self.assertEqual(ret, 0) + self.check_output() + + def test_arg_parse(self): + args = [ + '--get_initial_items', './gen_init_list.py', '--switch_to_good', + './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py', + '--test_script', './is_good.py', '--prune', '--file_args' + ] + ret = binary_search_state.Main(args) + self.assertEqual(ret, 0) + self.check_output() + + def test_test_setup_script(self): + os.remove('./is_setup') + with self.assertRaises(AssertionError): + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True) + + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + test_setup_script='./test_setup.py', + prune=True, + file_args=True) + self.assertEqual(ret, 0) + self.check_output() + + def test_bad_test_setup_script(self): + with self.assertRaises(AssertionError): + binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + test_setup_script='./test_setup_bad.py', + prune=True, + file_args=True) + + def test_bad_save_state(self): + state_file = binary_search_state.STATE_FILE + hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE) + + with open(state_file, 'w', encoding='utf-8') as f: + f.write('test123') + + bss = binary_search_state.MockBinarySearchState() + with self.assertRaises(binary_search_state.Error): + bss.SaveState() + + with open(state_file, 'r', encoding='utf-8') as f: + self.assertEqual(f.read(), 'test123') + + os.remove(state_file) + + # Cleanup generated save state that has no symlink + files = os.listdir(os.getcwd()) + save_states = [x for x in files if x.startswith(hidden_state_file)] + _ = [os.remove(x) for x in save_states] + + def test_save_state(self): + state_file = binary_search_state.STATE_FILE + + bss = binary_search_state.MockBinarySearchState() + bss.SaveState() + self.assertTrue(os.path.exists(state_file)) + first_state = os.readlink(state_file) + + bss.SaveState() + second_state = os.readlink(state_file) + self.assertTrue(os.path.exists(state_file)) + self.assertTrue(second_state != first_state) + self.assertFalse(os.path.exists(first_state)) + + bss.RemoveState() + self.assertFalse(os.path.islink(state_file)) + self.assertFalse(os.path.exists(second_state)) + + def test_load_state(self): + test_items = [1, 2, 3, 4, 5] + + bss = binary_search_state.MockBinarySearchState() + bss.all_items = test_items + bss.currently_good_items = set([1, 2, 3]) + bss.currently_bad_items = set([4, 5]) + bss.SaveState() + + bss = None + + bss2 = binary_search_state.MockBinarySearchState.LoadState() + self.assertEqual(bss2.all_items, test_items) + self.assertEqual(bss2.currently_good_items, set([])) + self.assertEqual(bss2.currently_bad_items, set([])) + + def test_tmp_cleanup(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='echo "0\n1\n2\n3"', + switch_to_good='./switch_tmp.py', + file_args=True) + bss.SwitchToGood(['0', '1', '2', '3']) + + tmp_file = None + with open('tmp_file', 'r', encoding='utf-8') as f: + tmp_file = f.read() + os.remove('tmp_file') + + self.assertFalse(os.path.exists(tmp_file)) + ws = common.ReadWorkingSet() + for i in range(3): + self.assertEqual(ws[i], 42) + + def test_verify_fail(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_bad.py', + switch_to_bad='./switch_to_good.py', + test_script='./is_good.py', + prune=True, + file_args=True, + verify=True) + with self.assertRaises(AssertionError): + bss.DoVerify() + + def test_early_terminate(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True, + iterations=1) + bss.DoSearchBadItems() + self.assertFalse(bss.found_items) + + def test_no_prune(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + test_setup_script='./test_setup.py', + prune=False, + file_args=True) + bss.DoSearchBadItems() + self.assertEqual(len(bss.found_items), 1) + + bad_objs = common.ReadObjectsFile() + found_obj = int(bss.found_items.pop()) + self.assertEqual(bad_objs[found_obj], 1) + + def test_set_file(self): + binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good_set_file.py', + switch_to_bad='./switch_to_bad_set_file.py', + test_script='./is_good.py', + prune=True, + file_args=True, + verify=True) + self.check_output() + + def test_noincremental_prune(self): + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good_noinc_prune.py', + switch_to_bad='./switch_to_bad_noinc_prune.py', + test_script='./is_good_noinc_prune.py', + test_setup_script='./test_setup.py', + prune=True, + noincremental=True, + file_args=True, + verify=False) + self.assertEqual(ret, 0) + self.check_output() + + def check_output(self): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad items are: " logs/binary_search_tool_test.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + + _, _, bad_ones = line.partition('Bad items are: ') + bad_ones = bad_ones.split() + expected_result = common.ReadObjectsFile() + + # Reconstruct objects file from bad_ones and compare + actual_result = [0] * len(expected_result) + for bad_obj in bad_ones: + actual_result[int(bad_obj)] = 1 + + self.assertEqual(actual_result, expected_result) + + +class BisectingUtilsPassTest(BisectingUtilsTest): + """Tests for bisecting tool at pass/transformation level.""" + + def check_pass_output(self, pass_name, pass_num, trans_num): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad pass: " logs/binary_search_tool_test.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + _, _, bad_info = line.partition('Bad pass: ') + actual_info = pass_name + ' at number ' + str(pass_num) + self.assertEqual(actual_info, bad_info) + + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad transformation number: ' + '" logs/binary_search_tool_test.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + _, _, bad_info = line.partition('Bad transformation number: ') + actual_info = str(trans_num) + self.assertEqual(actual_info, bad_info) + + def test_with_prune(self): + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=True, + file_args=True) + self.assertEqual(ret, 1) + + def test_gen_cmd_script(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + bss.DoSearchBadItems() + cmd_script_path = bss.cmd_script + self.assertTrue(os.path.exists(cmd_script_path)) + + def test_no_pass_support(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + bss.cmd_script = './cmd_script_no_support.py' + # No support for -opt-bisect-limit + with self.assertRaises(RuntimeError): + bss.BuildWithPassLimit(-1) + + def test_no_transform_support(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + bss.cmd_script = './cmd_script_no_support.py' + # No support for -print-debug-counter + with self.assertRaises(RuntimeError): + bss.BuildWithTransformLimit(-1, 'counter_name') + + def test_pass_transform_bisect(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + pass_num = 4 + trans_num = 19 + bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num) + bss.DoSearchBadPass() + self.check_pass_output('instcombine-visit', pass_num, trans_num) + + def test_result_not_reproduced_pass(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + # Fails reproducing at pass level. + pass_num = 0 + trans_num = 19 + bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num) + with self.assertRaises(ValueError): + bss.DoSearchBadPass() + + def test_result_not_reproduced_transform(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + pass_bisect='./generate_cmd.py', + prune=False, + file_args=True) + # Fails reproducing at transformation level. + pass_num = 4 + trans_num = 0 + bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num) + with self.assertRaises(ValueError): + bss.DoSearchBadPass() + + +class BisectStressTest(unittest.TestCase): + """Stress tests for bisecting tool.""" + + def test_every_obj_bad(self): + amt = 25 + gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)]) + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True, + verify=False) + self.assertEqual(ret, 0) + self.check_output() + + def test_every_index_is_bad(self): + amt = 25 + for i in range(amt): + obj_list = ['0'] * amt + obj_list[i] = '1' + obj_list = ','.join(obj_list) + gen_obj.Main(['--obj_list', obj_list]) + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_setup_script='./test_setup.py', + test_script='./is_good.py', + prune=True, + file_args=True) + self.assertEqual(ret, 0) + self.check_output() + + def check_output(self): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad items are: " logs/binary_search_tool_test.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + + _, _, bad_ones = line.partition('Bad items are: ') + bad_ones = bad_ones.split() + expected_result = common.ReadObjectsFile() + + # Reconstruct objects file from bad_ones and compare + actual_result = [0] * len(expected_result) + for bad_obj in bad_ones: + actual_result[int(bad_obj)] = 1 + + self.assertEqual(actual_result, expected_result) + + +def Main(argv): + num_tests = 2 + if len(argv) > 1: + num_tests = int(argv[1]) + + suite = unittest.TestSuite() + for _ in range(0, num_tests): + suite.addTest(BisectingUtilsTest()) + suite.addTest(BisectingUtilsTest('test_arg_parse')) + suite.addTest(BisectingUtilsTest('test_test_setup_script')) + suite.addTest(BisectingUtilsTest('test_bad_test_setup_script')) + suite.addTest(BisectingUtilsTest('test_bad_save_state')) + suite.addTest(BisectingUtilsTest('test_save_state')) + suite.addTest(BisectingUtilsTest('test_load_state')) + suite.addTest(BisectingUtilsTest('test_tmp_cleanup')) + suite.addTest(BisectingUtilsTest('test_verify_fail')) + suite.addTest(BisectingUtilsTest('test_early_terminate')) + suite.addTest(BisectingUtilsTest('test_no_prune')) + suite.addTest(BisectingUtilsTest('test_set_file')) + suite.addTest(BisectingUtilsTest('test_noincremental_prune')) + suite.addTest(BisectingUtilsPassTest('test_with_prune')) + suite.addTest(BisectingUtilsPassTest('test_gen_cmd_script')) + suite.addTest(BisectingUtilsPassTest('test_no_pass_support')) + suite.addTest(BisectingUtilsPassTest('test_no_transform_support')) + suite.addTest(BisectingUtilsPassTest('test_pass_transform_bisect')) + suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_pass')) + suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_transform')) + suite.addTest(BisectTest('test_full_bisector')) + suite.addTest(BisectStressTest('test_every_obj_bad')) + suite.addTest(BisectStressTest('test_every_index_is_bad')) + runner = unittest.TextTestRunner() + runner.run(suite) + + +if __name__ == '__main__': + Main(sys.argv) |