binary search tool: Properly cache all side effects

Properly cache all side effects in compiler wrapper. Extract some common
caching logic to their own functions (cache_file/restore_file).

TEST=Full Android bisection, and full NDK bisection with dwo files

Change-Id: I4c3be2cb5dae1922c145ba6be85ae65d1ed5cda2
Reviewed-on: https://chrome-internal-review.googlesource.com/273115
Commit-Ready: Cassidy Burden <cburden@google.com>
Tested-by: Cassidy Burden <cburden@google.com>
Reviewed-by: Caroline Tice <cmtice@google.com>

1 files changed, 99 insertions, 82 deletions

diff --git a/binary_search_tool/bisect_driver.py b/binary_search_tool/bisect_driver.py
index 2fa6760a..0e3d3998 100644
--- a/binary_search_tool/bisect_driver.py
+++ b/binary_search_tool/bisect_driver.py
@@ -22,7 +22,6 @@ import subprocess
 import sys
 
 VALID_MODES = ['POPULATE_GOOD', 'POPULATE_BAD', 'TRIAGE']
-DEP_CACHE = 'dep'
 GOOD_CACHE = 'good'
 BAD_CACHE = 'bad'
 LIST_FILE = os.path.join(GOOD_CACHE, '_LIST')
@@ -92,8 +91,8 @@ def exec_and_return(execargs):
   return subprocess.call(execargs)
 
 
-def in_bad_set(obj_file):
-  """Check if object file is in bad set.
+def which_cache(obj_file):
+  """Determine which cache an object belongs to.
 
   The binary search tool creates two files for each search iteration listing
   the full set of bad objects and full set of good objects. We use this to
@@ -101,7 +100,10 @@ def in_bad_set(obj_file):
   """
   bad_set_file = os.environ.get('BISECT_BAD_SET')
   ret = subprocess.call(['grep', '-x', '-q', obj_file, bad_set_file])
-  return ret == 0
+  if ret == 0:
+    return BAD_CACHE
+  else:
+    return GOOD_CACHE
 
 
 def makedirs(path):
@@ -117,40 +119,63 @@ def get_obj_path(execargs):
   """Get the object path for the object file in the list of arguments.
 
   Returns:
-    Tuple of object path from execution args (-o argument) and full object
-    path. If no object being outputted or output doesn't end in ".o" then return
-    empty strings.
+    Absolute object path from execution args (-o argument). If no object being
+    outputted or output doesn't end in ".o" then return empty string.
   """
   try:
     i = execargs.index('-o')
   except ValueError:
-    return '', ''
+    return ''
 
   obj_path = execargs[i + 1]
   if not obj_path.endswith(('.o',)):
     # TODO: what suffixes do we need to contemplate
     # TODO: add this as a warning
     # TODO: need to handle -r compilations
-    return '', ''
+    return ''
 
-  return obj_path, os.path.abspath(obj_path)
+  return os.path.abspath(obj_path)
 
 
 def get_dep_path(execargs):
   """Get the dep file path for the dep file in the list of arguments.
 
   Returns:
-    Tuple of dependency file path from execution args (-o argument) and full
-    dependency file path. If no dependency being outputted then return empty
-    strings.
+    Absolute path of dependency file path from execution args (-o argument). If
+    no dependency being outputted then return empty string.
   """
-  try:
+  if '-MD' not in execargs:
+    return ''
+
+  # If -MF given this is the path of the dependency file. Otherwise the
+  # dependency file is the value of -o but with a .d extension
+  if '-MF' in execargs:
     i = execargs.index('-MF')
-  except ValueError:
-    return '', ''
+    dep_path = execargs[i + 1]
+    return os.path.abspath(dep_path)
+
+  full_obj_path = get_obj_path(execargs)
+  if not full_obj_path:
+    return ''
+
+  return full_obj_path[:-2] + '.d'
+
+
+def get_dwo_path(execargs):
+  """Get the dwo file path for the dwo file in the list of arguments.
+
+  Returns:
+    Absolute dwo file path from execution args (-gsplit-dwarf argument) If no
+    dwo file being outputted then return empty string.
+  """
+  if '-gsplit-dwarf' not in execargs:
+    return ''
 
-  dep_path = execargs[i + 1]
-  return dep_path, os.path.abspath(dep_path)
+  full_obj_path = get_obj_path(execargs)
+  if not full_obj_path:
+    return ''
+
+  return full_obj_path[:-2] + '.dwo'
 
 
 def in_object_list(obj_name, list_filename):
@@ -166,34 +191,60 @@ def in_object_list(obj_name, list_filename):
     return False
 
 
-def generate_side_effects(execargs, bisect_dir):
-  """Generate compiler side effects.
+def get_side_effects(execargs):
+  """Determine side effects generated by compiler
 
-  Generate and cache side effects so that we can trick make into thinking
-  the compiler is actually called during triaging.
+  Returns:
+    List of paths of objects that the compiler generates as side effects.
   """
-  # TODO(cburden): Cache .dwo files
+  side_effects = []
 
   # Cache dependency files
-  dep_path, full_dep_path = get_dep_path(execargs)
-  if not dep_path:
-    return
+  full_dep_path = get_dep_path(execargs)
+  if full_dep_path:
+    side_effects.append(full_dep_path)
+
+  # Cache dwo files
+  full_dwo_path = get_dwo_path(execargs)
+  if full_dwo_path:
+    side_effects.append(full_dwo_path)
+
+  return side_effects
 
+
+def cache_file(execargs, bisect_dir, cache, abs_file_path):
+  """Cache compiler output file (.o/.d/.dwo)."""
   # os.path.join fails with absolute paths, use + instead
-  bisect_path = os.path.join(bisect_dir, DEP_CACHE) + full_dep_path
+  bisect_path = os.path.join(bisect_dir, cache) + abs_file_path
   bisect_path_dir = os.path.dirname(bisect_path)
   makedirs(bisect_path_dir)
-  pop_log = os.path.join(bisect_dir, DEP_CACHE, '_POPULATE_LOG')
-  log_to_file(pop_log, execargs, dep_path, bisect_path)
+  pop_log = os.path.join(bisect_dir, cache, '_POPULATE_LOG')
+  log_to_file(pop_log, execargs, abs_file_path, bisect_path)
 
   try:
-    if os.path.exists(dep_path):
-      shutil.copy2(dep_path, bisect_path)
+    if os.path.exists(abs_file_path):
+      shutil.copy2(abs_file_path, bisect_path)
+      # Set cache object to be read-only so later compilations can't
+      # accidentally overwrite it.
+      os.chmod(bisect_path, 0444)
   except Exception:
-    print('Could not get dep file', file=sys.stderr)
+    print('Could not cache file %s' % abs_file_path, file=sys.stderr)
     raise
 
 
+def restore_file(bisect_dir, cache, abs_file_path):
+  """Restore file from cache (.o/.d/.dwo)."""
+  # os.path.join fails with absolute paths, use + instead
+  cached_path = os.path.join(bisect_dir, cache) + abs_file_path
+  if os.path.exists(cached_path):
+    if os.path.exists(abs_file_path):
+      os.remove(abs_file_path)
+    os.link(cached_path, abs_file_path)
+  else:
+    raise Error(('%s is missing from %s cache! Unsure how to proceed. Make will'
+                 'now crash.' % (cache, cached_path)))
+
+
 def bisect_populate(execargs, bisect_dir, population_name):
   """Add necessary information to the bisect cache for the given execution.
 
@@ -211,44 +262,27 @@ def bisect_populate(execargs, bisect_dir, population_name):
   if retval:
     return retval
 
-  population_dir = os.path.join(bisect_dir, population_name)
-  makedirs(population_dir)
-  pop_log = os.path.join(population_dir, '_POPULATE_LOG')
-  log_to_file(pop_log, execargs)
-
-  obj_path, full_obj_path = get_obj_path(execargs)
-  if not obj_path:
+  full_obj_path = get_obj_path(execargs)
+  # If not a normal compiler call then just exit
+  if not full_obj_path:
     return
 
-  # os.path.join fails with absolute paths, use + instead
-  bisect_path = population_dir + full_obj_path
-  bisect_path_dir = os.path.dirname(bisect_path)
-  makedirs(bisect_path_dir)
-
-  try:
-    if os.path.exists(obj_path):
-      shutil.copy2(obj_path, bisect_path)
-      # Set cache object to be read-only so later compilations can't
-      # accidentally overwrite it.
-      os.chmod(bisect_path, 0444)
-  except Exception:
-    print('Could not populate bisect cache', file=sys.stderr)
-    raise
+  cache_file(execargs, bisect_dir, population_name, full_obj_path)
 
+  population_dir = os.path.join(bisect_dir, population_name)
   with lock_file(os.path.join(population_dir, '_LIST'), 'a') as object_list:
     object_list.write('%s\n' % full_obj_path)
 
-  # Cache the side effects generated by good compiler
-  if population_name == GOOD_CACHE:
-    generate_side_effects(execargs, bisect_dir)
+  for side_effect in get_side_effects(execargs):
+    cache_file(execargs, bisect_dir, population_name, side_effect)
 
 
 def bisect_triage(execargs, bisect_dir):
-  obj_path, full_obj_path = get_obj_path(execargs)
+  full_obj_path = get_obj_path(execargs)
   obj_list = os.path.join(bisect_dir, LIST_FILE)
 
   # If the output isn't an object file just call compiler
-  if not obj_path:
+  if not full_obj_path:
     return exec_and_return(execargs)
 
   # If this isn't a bisected object just call compiler
@@ -261,36 +295,19 @@ def bisect_triage(execargs, bisect_dir):
     else:
       raise Error(('%s is missing from cache! To ignore export '
                    'BISECT_CONTINUE_ON_MISSING=1. See documentation for more '
-                   'details on this option.' % obj_path))
+                   'details on this option.' % full_obj_path))
+
+  cache = which_cache(full_obj_path)
 
   # Generate compiler side effects. Trick Make into thinking compiler was
   # actually executed.
-
-  # If dependency is generated from this call, link it from dependency cache
-  dep_path, full_dep_path = get_dep_path(execargs)
-  if dep_path:
-    cached_dep_path = os.path.join(bisect_dir, DEP_CACHE) + dep_path
-    if os.path.exists(cached_dep_path):
-      if os.path.exists(full_dep_path):
-        os.remove(full_dep_path)
-      os.link(cached_dep_path, full_dep_path)
-    else:
-      raise Error(('%s is missing from dependency cache! Unsure how to '
-                   'proceed. Make will now crash.' % cached_dep_path))
+  for side_effect in get_side_effects(execargs):
+    restore_file(bisect_dir, cache, side_effect)
 
   # If generated object file happened to be pruned/cleaned by Make then link it
   # over from cache again.
-  if not os.path.exists(obj_path):
-    cache = BAD_CACHE if in_bad_set(full_obj_path) else GOOD_CACHE
-    cached_obj_path = os.path.join(bisect_dir, cache) + full_obj_path
-    if os.path.exists(cached_obj_path):
-      os.link(cached_obj_path, full_obj_path)
-    else:
-      raise Error('%s does not exist in %s cache' % (full_obj_path, cache))
-
-    # This is just used for debugging and stats gathering
-    log_file = os.path.join(bisect_dir, '_MISSING_OBJ_LOG')
-    log_to_file(log_file, execargs, cached_obj_path, full_obj_path)
+  if not os.path.exists(full_obj_path):
+    restore_file(bisect_dir, cache, full_obj_path)
 
 
 def bisect_driver(bisect_stage, bisect_dir, execargs):