bisect tool: Adding unit tests for pass/transform level bisect

This patch added unit tests for pass and transformation level bisect to
the bisecting tool. Since no compiler is involved in unit test, it used
cmd_script.py to simulate the output of compiler (to stderr).

BUG=chromium:878954
TEST=Passed all unit tests.

Change-Id: I599c079497333ec24f08f37c3315a16f28c0f887
Reviewed-on: https://chromium-review.googlesource.com/1298384
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Caroline Tice <cmtice@chromium.org>

5 files changed, 253 insertions, 20 deletions

diff --git a/binary_search_tool/test/binary_search_tool_tester.py b/binary_search_tool/test/binary_search_tool_tester.py
index 923ea112..aff45a86 100755
--- a/binary_search_tool/test/binary_search_tool_tester.py
+++ b/binary_search_tool/test/binary_search_tool_tester.py
@@ -132,7 +132,7 @@ class BisectingUtilsTest(unittest.TestCase):
 
     cleanup_list = [
         './is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad',
-        'noinc_prune_good'
+        'noinc_prune_good', './cmd_script.sh'
     ]
     for f in cleanup_list:
       if os.path.exists(f):
@@ -304,24 +304,6 @@ class BisectingUtilsTest(unittest.TestCase):
     found_obj = int(bss.found_items.pop())
     self.assertEquals(bad_objs[found_obj], 1)
 
-  def test_pass_bisect(self):
-    bss = binary_search_state.MockBinarySearchState(
-        get_initial_items='./gen_init_list.py',
-        switch_to_good='./switch_to_good.py',
-        switch_to_bad='./switch_to_bad.py',
-        pass_bisect='./generate_cmd.py',
-        test_script='./is_good.py',
-        test_setup_script='./test_setup.py',
-        prune=False,
-        file_args=True)
-    # TODO: Need to design unit tests for pass level bisection
-    bss.DoSearchBadItems()
-    self.assertEquals(len(bss.found_items), 1)
-
-    bad_objs = common.ReadObjectsFile()
-    found_obj = int(bss.found_items.pop())
-    self.assertEquals(bad_objs[found_obj], 1)
-
   def test_set_file(self):
     binary_search_state.Run(
         get_initial_items='./gen_init_list.py',
@@ -367,6 +349,131 @@ class BisectingUtilsTest(unittest.TestCase):
     self.assertEqual(actual_result, expected_result)
 
 
+class BisectingUtilsPassTest(BisectingUtilsTest):
+  """Tests for bisecting tool at pass/transformation level."""
+
+  def check_pass_output(self, pass_name, pass_num, trans_num):
+    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
+        ('grep "Bad pass: " logs/binary_search_tool_tester.py.out | '
+         'tail -n1'))
+    ls = out.splitlines()
+    self.assertEqual(len(ls), 1)
+    line = ls[0]
+    _, _, bad_info = line.partition('Bad pass: ')
+    actual_info = pass_name + ' at number ' + str(pass_num)
+    self.assertEqual(actual_info, bad_info)
+
+    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
+        ('grep "Bad transformation number: '
+         '" logs/binary_search_tool_tester.py.out | '
+         'tail -n1'))
+    ls = out.splitlines()
+    self.assertEqual(len(ls), 1)
+    line = ls[0]
+    _, _, bad_info = line.partition('Bad transformation number: ')
+    actual_info = str(trans_num)
+    self.assertEqual(actual_info, bad_info)
+
+  def test_with_prune(self):
+    ret = binary_search_state.Run(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=True,
+        file_args=True)
+    self.assertEquals(ret, 1)
+
+  def test_gen_cmd_script(self):
+    bss = binary_search_state.MockBinarySearchState(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=False,
+        file_args=True)
+    bss.DoSearchBadItems()
+    cmd_script_path = bss.cmd_script
+    self.assertTrue(os.path.exists(cmd_script_path))
+
+  def test_no_pass_support(self):
+    bss = binary_search_state.MockBinarySearchState(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=False,
+        file_args=True)
+    bss.cmd_script = './cmd_script_no_support.py'
+    # No support for -opt-bisect-limit
+    with self.assertRaises(RuntimeError):
+      bss.BuildWithPassLimit(-1)
+
+  def test_no_transform_support(self):
+    bss = binary_search_state.MockBinarySearchState(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=False,
+        file_args=True)
+    bss.cmd_script = './cmd_script_no_support.py'
+    # No support for -print-debug-counter
+    with self.assertRaises(RuntimeError):
+      bss.BuildWithTransformLimit(-1, 'counter_name')
+
+  def test_pass_transform_bisect(self):
+    bss = binary_search_state.MockBinarySearchState(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=False,
+        file_args=True)
+    pass_num = 4
+    trans_num = 19
+    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
+    bss.DoSearchBadPass()
+    self.check_pass_output('instcombine-visit', pass_num, trans_num)
+
+  def test_result_not_reproduced_pass(self):
+    bss = binary_search_state.MockBinarySearchState(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=False,
+        file_args=True)
+    # Fails reproducing at pass level.
+    pass_num = 0
+    trans_num = 19
+    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
+    with self.assertRaises(ValueError):
+      bss.DoSearchBadPass()
+
+  def test_result_not_reproduced_transform(self):
+    bss = binary_search_state.MockBinarySearchState(
+        get_initial_items='./gen_init_list.py',
+        switch_to_good='./switch_to_good.py',
+        switch_to_bad='./switch_to_bad.py',
+        test_script='./is_good.py',
+        pass_bisect='./generate_cmd.py',
+        prune=False,
+        file_args=True)
+    # Fails reproducing at transformation level.
+    pass_num = 4
+    trans_num = 0
+    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
+    with self.assertRaises(ValueError):
+      bss.DoSearchBadPass()
+
+
 class BisectStressTest(unittest.TestCase):
   """Stress tests for bisecting tool."""
 
@@ -442,6 +549,13 @@ def Main(argv):
   suite.addTest(BisectingUtilsTest('test_no_prune'))
   suite.addTest(BisectingUtilsTest('test_set_file'))
   suite.addTest(BisectingUtilsTest('test_noincremental_prune'))
+  suite.addTest(BisectingUtilsPassTest('test_with_prune'))
+  suite.addTest(BisectingUtilsPassTest('test_gen_cmd_script'))
+  suite.addTest(BisectingUtilsPassTest('test_no_pass_support'))
+  suite.addTest(BisectingUtilsPassTest('test_no_transform_support'))
+  suite.addTest(BisectingUtilsPassTest('test_pass_transform_bisect'))
+  suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_pass'))
+  suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_transform'))
   suite.addTest(BisectTest('test_full_bisector'))
   suite.addTest(BisectStressTest('test_every_obj_bad'))
   suite.addTest(BisectStressTest('test_every_index_is_bad'))
diff --git a/binary_search_tool/test/cmd_script.py b/binary_search_tool/test/cmd_script.py
new file mode 100755
index 00000000..6940eaae
--- /dev/null
+++ b/binary_search_tool/test/cmd_script.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python2
+"""Command script without compiler support for pass level bisection.
+
+This script generates a pseudo log which a workable compiler should print out.
+It assumes that -opt-bisect-limit and -print-debug-counter are supported by the
+compiler.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+
+import common
+
+
+def Main(argv):
+  if not os.path.exists('./is_setup'):
+    return 1
+
+  if len(argv) != 3:
+    return 1
+
+  limit_flags = os.environ['LIMIT_FLAGS']
+  opt_bisect_exist = False
+  debug_counter_exist = False
+
+  for option in limit_flags.split():
+    if '-opt-bisect-limit' in option:
+      opt_bisect_limit = int(option.split('=')[-1])
+      opt_bisect_exist = True
+    if '-debug-counter=' in option:
+      debug_counter = int(option.split('=')[-1])
+      debug_counter_exist = True
+
+  if not opt_bisect_exist:
+    return 1
+
+  # Manually set total number and bad number
+  total_pass = 10
+  total_transform = 20
+  bad_pass = int(argv[1])
+  bad_transform = int(argv[2])
+
+  if opt_bisect_limit == -1:
+    opt_bisect_limit = total_pass
+
+  for i in xrange(1, total_pass + 1):
+    bisect_str = 'BISECT: %srunning pass (%d) Combine redundant ' \
+                 'instructions on function (f1)' \
+                 % ('NOT ' if i > opt_bisect_limit else '', i)
+    print(bisect_str, file=sys.stderr)
+
+  if debug_counter_exist:
+    print('Counters and values:', file=sys.stderr)
+    print(
+        'instcombine-visit : {%d, 0, %d}' % (total_transform, debug_counter),
+        file=sys.stderr)
+
+  if opt_bisect_limit > bad_pass or \
+    (debug_counter_exist and debug_counter > bad_transform):
+    common.WriteWorkingSet([1])
+  else:
+    common.WriteWorkingSet([0])
+
+  return 0
+
+
+if __name__ == '__main__':
+  retval = Main(sys.argv)
+  sys.exit(retval)
diff --git a/binary_search_tool/test/cmd_script_no_support.py b/binary_search_tool/test/cmd_script_no_support.py
new file mode 100644
index 00000000..a817f300
--- /dev/null
+++ b/binary_search_tool/test/cmd_script_no_support.py
@@ -0,0 +1,23 @@
+"""Command script without compiler support for pass level bisection.
+
+This script generates a pseudo log when certain bisecting flags are not
+supported by compiler.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+
+
+def Main():
+  if not os.path.exists('./is_setup'):
+    return 1
+  print('No support for -opt-bisect-limit or -print-debug-counter.',
+        file=sys.stderr)
+  return 0
+
+
+if __name__ == '__main__':
+  retval = Main()
+  sys.exit(retval)
diff --git a/binary_search_tool/test/gen_obj.py b/binary_search_tool/test/gen_obj.py
index d17e93f5..a2bc7d93 100755
--- a/binary_search_tool/test/gen_obj.py
+++ b/binary_search_tool/test/gen_obj.py
@@ -85,7 +85,7 @@ def Main(argv):
   f.close()
 
   obj_num = len(obj_list)
-  bad_obj_num = obj_list.count('1')
+  bad_obj_num = obj_list.count(1)
   print('Generated {0} object files, with {1} bad ones.'.format(
       obj_num, bad_obj_num))
 
diff --git a/binary_search_tool/test/generate_cmd.py b/binary_search_tool/test/generate_cmd.py
new file mode 100755
index 00000000..f6876eda
--- /dev/null
+++ b/binary_search_tool/test/generate_cmd.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python2
+"""Generate a virtual cmd script for pass level bisection.
+
+This is a required argument for pass level bisecting. For unit test, we use
+this script to verify if cmd_script.sh is generated correctly.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+
+
+def Main():
+  if not os.path.exists('./is_setup'):
+    return 1
+  file_name = 'cmd_script.sh'
+  with open(file_name, 'w') as f:
+    f.write('Generated by generate_cmd.py')
+  return 0
+
+
+if __name__ == '__main__':
+  retval = Main()
+  sys.exit(retval)