1 files changed, 139 insertions, 0 deletions
diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py
new file mode 100755
index 00000000..e8ee2ab6
--- /dev/null
+++ b/llvm_tools/bisect_clang_crashes.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches and submits the artifacts from Chrome OS toolchain's crash bucket.
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import glob
+import json
+import logging
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+
+import chroot
+
+
+def get_artifacts(pattern):
+  results = subprocess.check_output(['gsutil.py', 'ls', pattern],
+                                    stderr=subprocess.STDOUT,
+                                    encoding='utf-8')
+  return sorted(l.strip() for l in results.splitlines())
+
+
+def get_crash_reproducers(working_dir):
+  results = []
+  for src in [
+      f for f in glob.glob('%s/*.c*' % working_dir)
+      if f.split('.')[-1] in ['c', 'cc', 'cpp']
+  ]:
+    script = '.'.join(src.split('.')[:-1]) + '.sh'
+    if not os.path.exists(script):
+      logging.warning('could not find the matching script of %s', src)
+    else:
+      results.append((src, script))
+  return results
+
+
+def submit_crash_to_forcey(forcey: str, temporary_directory: str,
+                           buildbucket_id: str, url: str) -> None:
+  dest_dir = os.path.join(temporary_directory, buildbucket_id)
+  dest_file = os.path.join(dest_dir, os.path.basename(url))
+  logging.info('Downloading and submitting %r...', url)
+  subprocess.check_output(['gsutil.py', 'cp', url, dest_file],
+                          stderr=subprocess.STDOUT)
+  subprocess.check_output(['tar', '-xJf', dest_file], cwd=dest_dir)
+  for src, script in get_crash_reproducers(dest_dir):
+    subprocess.check_output([
+        forcey, 'reduce', '-wait=false', '-note',
+        '%s:%s' % (url, src), '-sh_file', script, '-src_file', src
+    ])
+
+
+def main(argv):
+  chroot.VerifyOutsideChroot()
+  logging.basicConfig(
+      format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
+      level=logging.INFO,
+  )
+  cur_dir = os.path.dirname(os.path.abspath(__file__))
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument(
+      '--4c', dest='forcey', required=True, help='Path to a 4c client binary')
+  parser.add_argument(
+      '--state_file',
+      default=os.path.join(cur_dir, 'chromeos-state.json'),
+      help='The path to the state file.')
+  parser.add_argument(
+      '--nocleanup',
+      action='store_false',
+      dest='cleanup',
+      help='Keep temporary files created after the script finishes.')
+  opts = parser.parse_args(argv)
+
+  state_file = os.path.abspath(opts.state_file)
+  os.makedirs(os.path.dirname(state_file), exist_ok=True)
+  temporary_directory = '/tmp/bisect_clang_crashes'
+  os.makedirs(temporary_directory, exist_ok=True)
+  urls = get_artifacts('gs://chromeos-toolchain-artifacts/clang-crash-diagnoses'
+                       '/**/*clang_crash_diagnoses.tar.xz')
+  logging.info('%d crash URLs found', len(urls))
+
+  visited = {}
+  if os.path.exists(state_file):
+    buildbucket_ids = {url.split('/')[-2] for url in urls}
+    with open(state_file, encoding='utf-8') as f:
+      data = json.load(f)
+      visited = {k: v for k, v in data.items() if k in buildbucket_ids}
+    logging.info('Successfully loaded %d previously-submitted crashes',
+                 len(visited))
+
+  try:
+    for url in urls:
+      splits = url.split('/')
+      buildbucket_id = splits[-2]
+      # Skip the builds that has been processed
+      if buildbucket_id in visited:
+        continue
+      submit_crash_to_forcey(
+          forcey=opts.forcey,
+          temporary_directory=temporary_directory,
+          buildbucket_id=buildbucket_id,
+          url=url,
+      )
+      visited[buildbucket_id] = url
+
+    exception_in_flight = False
+  except:
+    exception_in_flight = True
+    raise
+  finally:
+    if exception_in_flight:
+      # This is best-effort. If the machine powers off or similar, we'll just
+      # resubmit the same crashes, which is suboptimal, but otherwise
+      # acceptable.
+      logging.error('Something went wrong; attempting to save our work...')
+    else:
+      logging.info('Persisting state...')
+
+    tmp_state_file = state_file + '.tmp'
+    with open(tmp_state_file, 'w', encoding='utf-8') as f:
+      json.dump(visited, f, indent=2)
+    os.rename(tmp_state_file, state_file)
+
+    logging.info('State successfully persisted')
+
+  if opts.cleanup:
+    shutil.rmtree(temporary_directory)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))