Bisect tool: Support print IR differences before and after bisection

This patch provides an option to print out IR differences before and
after pass/transformation level bisection.

This feature will help if user want to know what exactly does the bad
pass/transformation do to IR. It added two extra run with `-S` and
`-emit-llvm` to generate two IR files and use `diff` to compare them.

Note: `--ir_diff` option only works when pass_bisect is enabled.

BUG=chromium:878954
TEST=Ran test successfully with Android compiler wrapper.

Change-Id: Ia691dc6f23331a3e584eaf9e390edeeb990f3c87
Reviewed-on: https://chromium-review.googlesource.com/1220015
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Caroline Tice <cmtice@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>

2 files changed, 78 insertions, 5 deletions

diff --git a/binary_search_tool/binary_search_state.py b/binary_search_tool/binary_search_state.py
index 0d5810c3..f6c8ac7c 100755
--- a/binary_search_tool/binary_search_state.py
+++ b/binary_search_tool/binary_search_state.py
@@ -14,6 +14,7 @@ import math
 import os
 import pickle
 import re
+import shutil
 import sys
 import tempfile
 import time
@@ -70,7 +71,8 @@ class BinarySearchState(object):
 
   def __init__(self, get_initial_items, switch_to_good, switch_to_bad,
                test_setup_script, test_script, incremental, prune, pass_bisect,
-               iterations, prune_iterations, verify, file_args, verbose):
+               ir_diff, iterations, prune_iterations, verify, file_args,
+               verbose):
     """BinarySearchState constructor, see Run for full args documentation."""
     self.get_initial_items = get_initial_items
     self.switch_to_good = switch_to_good
@@ -80,6 +82,7 @@ class BinarySearchState(object):
     self.incremental = incremental
     self.prune = prune
     self.pass_bisect = pass_bisect
+    self.ir_diff = ir_diff
     self.iterations = iterations
     self.prune_iterations = prune_iterations
     self.verify = verify
@@ -345,7 +348,7 @@ class BinarySearchState(object):
     # If pass not found, return None
     return None
 
-  def BuildWithPassLimit(self, limit):
+  def BuildWithPassLimit(self, limit, generate_ir=False):
     """ Rebuild bad item with pass level bisect limit
 
     Run command line script generated by GenerateBadCommandScript(), with
@@ -357,6 +360,8 @@ class BinarySearchState(object):
       pass_name: The debugcounter name of current limit pass.
     """
     os.environ['LIMIT_FLAGS'] = '-mllvm -opt-bisect-limit=' + str(limit)
+    if generate_ir:
+      os.environ['LIMIT_FLAGS'] += ' -S -emit-llvm'
     self.l.LogOutput(
         'Limit flags: %s' % os.environ['LIMIT_FLAGS'],
         print_to_console=self.verbose)
@@ -388,7 +393,11 @@ class BinarySearchState(object):
       raise ValueError('[Error] While building, limit number does not match.')
     return pass_num, self.CollectPassName(last_pass)
 
-  def BuildWithTransformLimit(self, limit, pass_name=None, pass_limit=-1):
+  def BuildWithTransformLimit(self,
+                              limit,
+                              pass_name=None,
+                              pass_limit=-1,
+                              generate_ir=False):
     """ Rebuild bad item with transformation level bisect limit
 
     Run command line script generated by GenerateBadCommandScript(), with
@@ -407,6 +416,8 @@ class BinarySearchState(object):
                                 ' -mllvm -debug-counter=' + counter_name + \
                                 '-count=' + str(limit) + \
                                 ' -mllvm -print-debug-counter'
+    if generate_ir:
+      os.environ['LIMIT_FLAGS'] += ' -S -emit-llvm'
     self.l.LogOutput(
         'Limit flags: %s' % os.environ['LIMIT_FLAGS'],
         print_to_console=self.verbose)
@@ -414,6 +425,10 @@ class BinarySearchState(object):
     _, _, msg = self.ce.RunCommandWOutput(command, print_to_console=False)
 
     if 'Counters and values:' not in msg:
+      # Print pass level IR diff only if transformation level bisection does
+      # not work.
+      if self.ir_diff:
+        self.PrintIRDiff(pass_limit)
       raise RuntimeError('No bisect info printed, DebugCounter may not be '
                          'supported by the compiler.')
 
@@ -439,6 +454,35 @@ class BinarySearchState(object):
     # transformation count.
     return 0
 
+  def PrintIRDiff(self, pass_index, pass_name=None, trans_index=-1):
+    bad_item = list(self.found_items)[0]
+    self.l.LogOutput(
+        'IR difference before and after bad pass/transformation:',
+        print_to_console=self.verbose)
+
+    if trans_index == -1:
+      # Pass level IR diff
+      self.BuildWithPassLimit(pass_index, self.ir_diff)
+      good_ir = os.path.join(tempfile.tempdir, 'good.s')
+      shutil.copyfile(bad_item, good_ir)
+      pass_index += 1
+      self.BuildWithPassLimit(pass_index, self.ir_diff)
+    else:
+      # Transformation level IR diff
+      self.BuildWithTransformLimit(trans_index, pass_name, pass_index,
+                                   self.ir_diff)
+      good_ir = os.path.join(tempfile.tempdir, 'good.s')
+      shutil.copyfile(bad_item, good_ir)
+      trans_index += 1
+      self.BuildWithTransformLimit(trans_index, pass_name, pass_index,
+                                   self.ir_diff)
+
+    bad_ir = os.path.join(tempfile.tempdir, 'bad.s')
+    shutil.copyfile(bad_item, bad_ir)
+
+    command = 'diff %s %s' % (good_ir, bad_ir)
+    _, _, _ = self.ce.RunCommandWOutput(command, print_to_console=self.verbose)
+
   def DoSearchBadPass(self):
     """Perform full search for bad pass of bad item."""
     logger.GetLogger().LogOutput('Starting to bisect bad pass for bad item.')
@@ -471,6 +515,10 @@ class BinarySearchState(object):
     trans_index, _ = self.DoBinarySearchBadPass(pass_index, pass_name)
     if (trans_index == 0):
       raise ValueError('Bisecting %s cannot reproduce good result.' % pass_name)
+
+    if self.ir_diff:
+      self.PrintIRDiff(pass_index, pass_name, trans_index)
+
     logger.GetLogger().LogOutput(
         'Bisection result for bad item %s:\n'
         'Bad pass: %s at number %d\n'
@@ -699,6 +747,7 @@ class MockBinarySearchState(BinarySearchState):
         'incremental': True,
         'prune': False,
         'pass_bisect': None,
+        'ir_diff': False,
         'iterations': 50,
         'prune_iterations': 100,
         'verify': True,
@@ -731,6 +780,7 @@ def Run(get_initial_items,
         iterations=50,
         prune=False,
         pass_bisect=None,
+        ir_diff=False,
         noincremental=False,
         file_args=False,
         verify=True,
@@ -760,6 +810,9 @@ def Run(get_initial_items,
       pass/ transformation level bisection for the bad item. Requires that
       'prune' be set to False, and needs support of `-opt-bisect-limit`(pass)
       and `-print-debug-counter`(transformation) from LLVM.
+    ir_diff: Whether to print IR differences before and after bad
+      pass/transformation to verbose output. Defaults to False, only works when
+      pass_bisect is enabled.
     noincremental: Whether to send "diffs" of good/bad items to switch scripts.
     file_args: If True then arguments to switch scripts will be a file name
       containing a newline separated list of the items to switch.
@@ -796,6 +849,10 @@ def Run(get_initial_items,
       logger.GetLogger().LogOutput('"--pass_bisect" only works when '
                                    '"--prune" is set to be False.')
       return 1
+    if not pass_bisect and ir_diff:
+      logger.GetLogger().LogOutput('"--ir_diff" only works when '
+                                   '"--pass_bisect" is enabled.')
+
     switch_to_good = _CanonicalizeScript(switch_to_good)
     switch_to_bad = _CanonicalizeScript(switch_to_bad)
     if test_setup_script:
@@ -810,8 +867,8 @@ def Run(get_initial_items,
 
     bss = BinarySearchState(get_initial_items, switch_to_good, switch_to_bad,
                             test_setup_script, test_script, incremental, prune,
-                            pass_bisect, iterations, prune_iterations, verify,
-                            file_args, verbose)
+                            pass_bisect, ir_diff, iterations, prune_iterations,
+                            verify, file_args, verbose)
     bss.DoVerify()
 
   try:
diff --git a/binary_search_tool/common.py b/binary_search_tool/common.py
index 2850801c..40660b52 100644
--- a/binary_search_tool/common.py
+++ b/binary_search_tool/common.py
@@ -203,6 +203,22 @@ def _BuildArgsDict(args):
            'For now it only supports one single bad item, so to use it, '
            'prune must be set to False.')
   # No input (evals to False),
+  # --ir_diff (evals to True),
+  # --ir_diff=False,
+  # --ir_diff=True
+  args.AddArgument(
+      '-d',
+      '--ir_diff',
+      dest='ir_diff',
+      nargs='?',
+      const=True,
+      default=False,
+      type=StrToBool,
+      metavar='bool',
+      help='Whether to print IR differences before and after bad '
+           'pass/transformation to verbose output. Defaults to False, '
+           'only works when pass_bisect is enabled.')
+  # No input (evals to False),
   # --noincremental (evals to True),
   # --noincremental=False,
   # --noincremental=True