diff options
Diffstat (limited to 'binary_search_tool/test/binary_search_tool_tester.py')
-rwxr-xr-x | binary_search_tool/test/binary_search_tool_tester.py | 421 |
1 files changed, 421 insertions, 0 deletions
diff --git a/binary_search_tool/test/binary_search_tool_tester.py b/binary_search_tool/test/binary_search_tool_tester.py new file mode 100755 index 00000000..775c1715 --- /dev/null +++ b/binary_search_tool/test/binary_search_tool_tester.py @@ -0,0 +1,421 @@ +#!/usr/bin/python2 + +# Copyright 2012 Google Inc. All Rights Reserved. +"""Tests for bisecting tool.""" + +from __future__ import print_function + +__author__ = 'shenhan@google.com (Han Shen)' + +import os +import random +import sys +import unittest + +from cros_utils import command_executer +from binary_search_tool import binary_search_state +from binary_search_tool import bisect + +import common +import gen_obj + + +def GenObj(): + obj_num = random.randint(100, 1000) + bad_obj_num = random.randint(obj_num / 100, obj_num / 20) + if bad_obj_num == 0: + bad_obj_num = 1 + gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)]) + + +def CleanObj(): + os.remove(common.OBJECTS_FILE) + os.remove(common.WORKING_SET_FILE) + print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE, + common.WORKING_SET_FILE)) + + +class BisectTest(unittest.TestCase): + """Tests for bisect.py""" + + def setUp(self): + with open('./is_setup', 'w'): + pass + + try: + os.remove(binary_search_state.STATE_FILE) + except OSError: + pass + + def tearDown(self): + try: + os.remove('./is_setup') + os.remove(os.readlink(binary_search_state.STATE_FILE)) + os.remove(binary_search_state.STATE_FILE) + except OSError: + pass + + class FullBisector(bisect.Bisector): + """Test bisector to test bisect.py with""" + + def __init__(self, options, overrides): + super(BisectTest.FullBisector, self).__init__(options, overrides) + + def PreRun(self): + GenObj() + return 0 + + def Run(self): + return binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True) + + def PostRun(self): + CleanObj() + return 0 + + def test_full_bisector(self): + ret = bisect.Run(self.FullBisector({}, {})) + self.assertEquals(ret, 0) + self.assertFalse(os.path.exists(common.OBJECTS_FILE)) + self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) + + def check_output(self): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + + _, _, bad_ones = line.partition('Bad items are: ') + bad_ones = bad_ones.split() + expected_result = common.ReadObjectsFile() + + # Reconstruct objects file from bad_ones and compare + actual_result = [0] * len(expected_result) + for bad_obj in bad_ones: + actual_result[int(bad_obj)] = 1 + + self.assertEqual(actual_result, expected_result) + + +class BisectingUtilsTest(unittest.TestCase): + """Tests for bisecting tool.""" + + def setUp(self): + """Generate [100-1000] object files, and 1-5% of which are bad ones.""" + GenObj() + + with open('./is_setup', 'w'): + pass + + try: + os.remove(binary_search_state.STATE_FILE) + except OSError: + pass + + def tearDown(self): + """Cleanup temp files.""" + CleanObj() + + try: + os.remove(os.readlink(binary_search_state.STATE_FILE)) + except OSError: + pass + + cleanup_list = ['./is_setup', binary_search_state.STATE_FILE, + 'noinc_prune_bad', 'noinc_prune_good'] + for f in cleanup_list: + if os.path.exists(f): + os.remove(f) + + def runTest(self): + ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True) + self.assertEquals(ret, 0) + self.check_output() + + def test_arg_parse(self): + args = ['--get_initial_items', './gen_init_list.py', '--switch_to_good', + './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py', + '--test_script', './is_good.py', '--prune', '--file_args'] + ret = binary_search_state.Main(args) + self.assertEquals(ret, 0) + self.check_output() + + def test_test_setup_script(self): + os.remove('./is_setup') + with self.assertRaises(AssertionError): + ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True) + + ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + test_setup_script='./test_setup.py', + prune=True, + file_args=True) + self.assertEquals(ret, 0) + self.check_output() + + def test_bad_test_setup_script(self): + with self.assertRaises(AssertionError): + binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + test_setup_script='./test_setup_bad.py', + prune=True, + file_args=True) + + def test_bad_save_state(self): + state_file = binary_search_state.STATE_FILE + hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE) + + with open(state_file, 'w') as f: + f.write('test123') + + bss = binary_search_state.MockBinarySearchState() + with self.assertRaises(binary_search_state.Error): + bss.SaveState() + + with open(state_file, 'r') as f: + self.assertEquals(f.read(), 'test123') + + os.remove(state_file) + + # Cleanup generated save state that has no symlink + files = os.listdir(os.getcwd()) + save_states = [x for x in files if x.startswith(hidden_state_file)] + _ = [os.remove(x) for x in save_states] + + def test_save_state(self): + state_file = binary_search_state.STATE_FILE + + bss = binary_search_state.MockBinarySearchState() + bss.SaveState() + self.assertTrue(os.path.exists(state_file)) + first_state = os.readlink(state_file) + + bss.SaveState() + second_state = os.readlink(state_file) + self.assertTrue(os.path.exists(state_file)) + self.assertTrue(second_state != first_state) + self.assertFalse(os.path.exists(first_state)) + + bss.RemoveState() + self.assertFalse(os.path.islink(state_file)) + self.assertFalse(os.path.exists(second_state)) + + def test_load_state(self): + test_items = [1, 2, 3, 4, 5] + + bss = binary_search_state.MockBinarySearchState() + bss.all_items = test_items + bss.currently_good_items = set([1, 2, 3]) + bss.currently_bad_items = set([4, 5]) + bss.SaveState() + + bss = None + + bss2 = binary_search_state.MockBinarySearchState.LoadState() + self.assertEquals(bss2.all_items, test_items) + self.assertEquals(bss2.currently_good_items, set([])) + self.assertEquals(bss2.currently_bad_items, set([])) + + def test_tmp_cleanup(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='echo "0\n1\n2\n3"', + switch_to_good='./switch_tmp.py', + file_args=True) + bss.SwitchToGood(['0', '1', '2', '3']) + + tmp_file = None + with open('tmp_file', 'r') as f: + tmp_file = f.read() + os.remove('tmp_file') + + self.assertFalse(os.path.exists(tmp_file)) + ws = common.ReadWorkingSet() + for i in range(3): + self.assertEquals(ws[i], 42) + + def test_verify_fail(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_bad.py', + switch_to_bad='./switch_to_good.py', + test_script='./is_good.py', + prune=True, + file_args=True, + verify=True) + with self.assertRaises(AssertionError): + bss.DoVerify() + + def test_early_terminate(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True, + iterations=1) + bss.DoSearch() + self.assertFalse(bss.found_items) + + def test_no_prune(self): + bss = binary_search_state.MockBinarySearchState( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + test_setup_script='./test_setup.py', + prune=False, + file_args=True) + bss.DoSearch() + self.assertEquals(len(bss.found_items), 1) + + bad_objs = common.ReadObjectsFile() + found_obj = int(bss.found_items.pop()) + self.assertEquals(bad_objs[found_obj], 1) + + def test_set_file(self): + binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good_set_file.py', + switch_to_bad='./switch_to_bad_set_file.py', + test_script='./is_good.py', + prune=True, + file_args=True, + verify=True) + self.check_output() + + def test_noincremental_prune(self): + ret = binary_search_state.Run( + get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good_noinc_prune.py', + switch_to_bad='./switch_to_bad_noinc_prune.py', + test_script='./is_good_noinc_prune.py', + test_setup_script='./test_setup.py', + prune=True, + noincremental=True, + file_args=True, + verify=False) + self.assertEquals(ret, 0) + self.check_output() + + def check_output(self): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + + _, _, bad_ones = line.partition('Bad items are: ') + bad_ones = bad_ones.split() + expected_result = common.ReadObjectsFile() + + # Reconstruct objects file from bad_ones and compare + actual_result = [0] * len(expected_result) + for bad_obj in bad_ones: + actual_result[int(bad_obj)] = 1 + + self.assertEqual(actual_result, expected_result) + + +class BisectStressTest(unittest.TestCase): + """Stress tests for bisecting tool.""" + + def test_every_obj_bad(self): + amt = 25 + gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)]) + ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_script='./is_good.py', + prune=True, + file_args=True, + verify=False) + self.assertEquals(ret, 0) + self.check_output() + + def test_every_index_is_bad(self): + amt = 25 + for i in range(amt): + obj_list = ['0'] * amt + obj_list[i] = '1' + obj_list = ','.join(obj_list) + gen_obj.Main(['--obj_list', obj_list]) + ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', + switch_to_good='./switch_to_good.py', + switch_to_bad='./switch_to_bad.py', + test_setup_script='./test_setup.py', + test_script='./is_good.py', + prune=True, + file_args=True) + self.assertEquals(ret, 0) + self.check_output() + + def check_output(self): + _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( + ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' + 'tail -n1')) + ls = out.splitlines() + self.assertEqual(len(ls), 1) + line = ls[0] + + _, _, bad_ones = line.partition('Bad items are: ') + bad_ones = bad_ones.split() + expected_result = common.ReadObjectsFile() + + # Reconstruct objects file from bad_ones and compare + actual_result = [0] * len(expected_result) + for bad_obj in bad_ones: + actual_result[int(bad_obj)] = 1 + + self.assertEqual(actual_result, expected_result) + + +def Main(argv): + num_tests = 2 + if len(argv) > 1: + num_tests = int(argv[1]) + + suite = unittest.TestSuite() + for _ in range(0, num_tests): + suite.addTest(BisectingUtilsTest()) + suite.addTest(BisectingUtilsTest('test_arg_parse')) + suite.addTest(BisectingUtilsTest('test_test_setup_script')) + suite.addTest(BisectingUtilsTest('test_bad_test_setup_script')) + suite.addTest(BisectingUtilsTest('test_bad_save_state')) + suite.addTest(BisectingUtilsTest('test_save_state')) + suite.addTest(BisectingUtilsTest('test_load_state')) + suite.addTest(BisectingUtilsTest('test_tmp_cleanup')) + suite.addTest(BisectingUtilsTest('test_verify_fail')) + suite.addTest(BisectingUtilsTest('test_early_terminate')) + suite.addTest(BisectingUtilsTest('test_no_prune')) + suite.addTest(BisectingUtilsTest('test_set_file')) + suite.addTest(BisectingUtilsTest('test_noincremental_prune')) + suite.addTest(BisectTest('test_full_bisector')) + suite.addTest(BisectStressTest('test_every_obj_bad')) + suite.addTest(BisectStressTest('test_every_index_is_bad')) + runner = unittest.TextTestRunner() + runner.run(suite) + + +if __name__ == '__main__': + Main(sys.argv) |