llvm_tools: add a tool to bisect clang crashes

Add a tool that downloads clang crash diagnoses from
chrome-toolchain-artifacts/clang-crash-dignoses and send them to 4c
server for bisection.

BUG=chromium:1056904
TEST=local tests

Change-Id: I75081259705c236a20173fc591d080c6b0504cc5
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2325008
Tested-by: Jian Cai <jiancai@google.com>
Reviewed-by: George Burgess <gbiv@chromium.org>

1 files changed, 109 insertions, 0 deletions

diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py
new file mode 100755
index 00000000..e7ce96e4
--- /dev/null
+++ b/llvm_tools/bisect_clang_crashes.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches and submits the artifacts from Chrome OS toolchain's crash bucket.
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import glob
+import json
+import logging
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+
+import chroot
+
+
+def get_artifacts(pattern):
+  results = subprocess.check_output(['gsutil', 'ls', pattern],
+                                    stderr=subprocess.STDOUT,
+                                    encoding='utf-8')
+  return [l.strip() for l in results.splitlines()]
+
+
+def get_crash_reproducers(working_dir):
+  results = []
+  for src in [
+      f for f in glob.glob('%s/*.c*' % working_dir)
+      if f.split('.')[-1] in ['c', 'cc', 'cpp']
+  ]:
+    script = '.'.join(src.split('.')[:-1]) + '.sh'
+    if not os.path.exists(script):
+      logging.warning('could not find the matching script of %s', src)
+    else:
+      results.append((src, script))
+  return results
+
+
+def main(argv):
+  chroot.VerifyOutsideChroot()
+  logging.basicConfig(
+      format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
+      level=logging.INFO,
+  )
+  cur_dir = os.path.dirname(os.path.abspath(__file__))
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument(
+      '--4c', dest='forcey', required=True, help='Path to a 4c client binary')
+  parser.add_argument(
+      '--state_file',
+      default=os.path.join(cur_dir, 'chromeos-state.json'),
+      help='The path to the state file.')
+  parser.add_argument(
+      '--nocleanup',
+      action='store_false',
+      dest='cleanup',
+      help='Keep temporary files created after the script finishes.')
+  opts = parser.parse_args(argv)
+
+  state_file = os.path.abspath(opts.state_file)
+  os.makedirs(os.path.dirname(state_file), exist_ok=True)
+  temporary_directory = '/tmp/bisect_clang_crashes'
+  os.makedirs(temporary_directory, exist_ok=True)
+  urls = get_artifacts('gs://chromeos-toolchain-artifacts/clang-crash-diagnoses'
+                       '/**/*clang_crash_diagnoses.tar.xz')
+  visited = {}
+  if os.path.exists(state_file):
+    buildbucket_ids = {url.split('/')[-2] for url in urls}
+    with open(state_file, encoding='utf-8') as f:
+      data = json.load(f)
+      visited = {k: v for k, v in data.items() if k in buildbucket_ids}
+  for url in urls:
+    splits = url.split('/')
+    buildbucket_id = splits[-2]
+    # Skip the builds that has been processed
+    if buildbucket_id in visited:
+      continue
+    visited[buildbucket_id] = '%s' % url
+    dest_dir = os.path.join(temporary_directory, buildbucket_id)
+    dest_file = os.path.join(dest_dir, splits[-1])
+    subprocess.check_output(['gsutil', 'cp', url, dest_file])
+    subprocess.check_output(['tar', '-xJf', dest_file], cwd=dest_dir)
+
+    for src, script in get_crash_reproducers(dest_dir):
+      subprocess.check_output(
+          [
+              opts.forcey, 'reduce', '-wait=false', '-note',
+              '%s:%s' % (url, src), '-sh_file', script, '-src_file', src
+          ],
+          encoding='utf-8',
+      )
+
+  tmp_state_file = state_file + '.tmp'
+  with open(tmp_state_file, 'w', encoding='utf-8') as f:
+    json.dump(visited, f, indent=2)
+  os.rename(tmp_state_file, state_file)
+  if opts.cleanup:
+    shutil.rmtree(temporary_directory)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))