diff options
Diffstat (limited to 'llvm_tools/bisect_clang_crashes.py')
| -rwxr-xr-x | llvm_tools/bisect_clang_crashes.py | 240 |
1 files changed, 130 insertions, 110 deletions
diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py index c53db179..b2759051 100755 --- a/llvm_tools/bisect_clang_crashes.py +++ b/llvm_tools/bisect_clang_crashes.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Fetches and submits the artifacts from Chrome OS toolchain's crash bucket. +"""Fetches and submits the artifacts from ChromeOS toolchain's crash bucket. """ import argparse @@ -21,117 +21,137 @@ import chroot def get_artifacts(pattern): - results = subprocess.check_output(['gsutil.py', 'ls', pattern], - stderr=subprocess.STDOUT, - encoding='utf-8') - return sorted(l.strip() for l in results.splitlines()) + results = subprocess.check_output( + ["gsutil.py", "ls", pattern], stderr=subprocess.STDOUT, encoding="utf-8" + ) + return sorted(l.strip() for l in results.splitlines()) def get_crash_reproducers(working_dir): - results = [] - for src in [ - f for f in glob.glob('%s/*.c*' % working_dir) - if f.split('.')[-1] in ['c', 'cc', 'cpp'] - ]: - script = '.'.join(src.split('.')[:-1]) + '.sh' - if not os.path.exists(script): - logging.warning('could not find the matching script of %s', src) - else: - results.append((src, script)) - return results - - -def submit_crash_to_forcey(forcey: str, temporary_directory: str, - buildbucket_id: str, url: str) -> None: - dest_dir = os.path.join(temporary_directory, buildbucket_id) - dest_file = os.path.join(dest_dir, os.path.basename(url)) - logging.info('Downloading and submitting %r...', url) - subprocess.check_output(['gsutil.py', 'cp', url, dest_file], - stderr=subprocess.STDOUT) - subprocess.check_output(['tar', '-xJf', dest_file], cwd=dest_dir) - for src, script in get_crash_reproducers(dest_dir): - subprocess.check_output([ - forcey, 'reduce', '-wait=false', '-note', - '%s:%s' % (url, src), '-sh_file', script, '-src_file', src - ]) + results = [] + for src in [ + f + for f in glob.glob("%s/*.c*" % working_dir) + if f.split(".")[-1] in ["c", "cc", "cpp"] + ]: + script = ".".join(src.split(".")[:-1]) + ".sh" + if not os.path.exists(script): + logging.warning("could not find the matching script of %s", src) + else: + results.append((src, script)) + return results + + +def submit_crash_to_forcey( + forcey: str, temporary_directory: str, buildbucket_id: str, url: str +) -> None: + dest_dir = os.path.join(temporary_directory, buildbucket_id) + dest_file = os.path.join(dest_dir, os.path.basename(url)) + logging.info("Downloading and submitting %r...", url) + subprocess.check_output( + ["gsutil.py", "cp", url, dest_file], stderr=subprocess.STDOUT + ) + subprocess.check_output(["tar", "-xJf", dest_file], cwd=dest_dir) + for src, script in get_crash_reproducers(dest_dir): + subprocess.check_output( + [ + forcey, + "reduce", + "-wait=false", + "-note", + "%s:%s" % (url, src), + "-sh_file", + script, + "-src_file", + src, + ] + ) def main(argv): - chroot.VerifyOutsideChroot() - logging.basicConfig( - format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s', - level=logging.INFO, - ) - cur_dir = os.path.dirname(os.path.abspath(__file__)) - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--4c', dest='forcey', required=True, help='Path to a 4c client binary') - parser.add_argument( - '--state_file', - default=os.path.join(cur_dir, 'chromeos-state.json'), - help='The path to the state file.') - parser.add_argument( - '--nocleanup', - action='store_false', - dest='cleanup', - help='Keep temporary files created after the script finishes.') - opts = parser.parse_args(argv) - - state_file = os.path.abspath(opts.state_file) - os.makedirs(os.path.dirname(state_file), exist_ok=True) - temporary_directory = '/tmp/bisect_clang_crashes' - os.makedirs(temporary_directory, exist_ok=True) - urls = get_artifacts('gs://chromeos-toolchain-artifacts/clang-crash-diagnoses' - '/**/*clang_crash_diagnoses.tar.xz') - logging.info('%d crash URLs found', len(urls)) - - visited = {} - if os.path.exists(state_file): - buildbucket_ids = {url.split('/')[-2] for url in urls} - with open(state_file, encoding='utf-8') as f: - data = json.load(f) - visited = {k: v for k, v in data.items() if k in buildbucket_ids} - logging.info('Successfully loaded %d previously-submitted crashes', - len(visited)) - - try: - for url in urls: - splits = url.split('/') - buildbucket_id = splits[-2] - # Skip the builds that has been processed - if buildbucket_id in visited: - continue - submit_crash_to_forcey( - forcey=opts.forcey, - temporary_directory=temporary_directory, - buildbucket_id=buildbucket_id, - url=url, - ) - visited[buildbucket_id] = url - - exception_in_flight = False - except: - exception_in_flight = True - raise - finally: - if exception_in_flight: - # This is best-effort. If the machine powers off or similar, we'll just - # resubmit the same crashes, which is suboptimal, but otherwise - # acceptable. - logging.error('Something went wrong; attempting to save our work...') - else: - logging.info('Persisting state...') - - tmp_state_file = state_file + '.tmp' - with open(tmp_state_file, 'w', encoding='utf-8') as f: - json.dump(visited, f, indent=2) - os.rename(tmp_state_file, state_file) - - logging.info('State successfully persisted') - - if opts.cleanup: - shutil.rmtree(temporary_directory) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + chroot.VerifyOutsideChroot() + logging.basicConfig( + format="%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s", + level=logging.INFO, + ) + cur_dir = os.path.dirname(os.path.abspath(__file__)) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--4c", dest="forcey", required=True, help="Path to a 4c client binary" + ) + parser.add_argument( + "--state_file", + default=os.path.join(cur_dir, "chromeos-state.json"), + help="The path to the state file.", + ) + parser.add_argument( + "--nocleanup", + action="store_false", + dest="cleanup", + help="Keep temporary files created after the script finishes.", + ) + opts = parser.parse_args(argv) + + state_file = os.path.abspath(opts.state_file) + os.makedirs(os.path.dirname(state_file), exist_ok=True) + temporary_directory = "/tmp/bisect_clang_crashes" + os.makedirs(temporary_directory, exist_ok=True) + urls = get_artifacts( + "gs://chromeos-toolchain-artifacts/clang-crash-diagnoses" + "/**/*clang_crash_diagnoses.tar.xz" + ) + logging.info("%d crash URLs found", len(urls)) + + visited = {} + if os.path.exists(state_file): + buildbucket_ids = {url.split("/")[-2] for url in urls} + with open(state_file, encoding="utf-8") as f: + data = json.load(f) + visited = {k: v for k, v in data.items() if k in buildbucket_ids} + logging.info( + "Successfully loaded %d previously-submitted crashes", len(visited) + ) + + try: + for url in urls: + splits = url.split("/") + buildbucket_id = splits[-2] + # Skip the builds that has been processed + if buildbucket_id in visited: + continue + submit_crash_to_forcey( + forcey=opts.forcey, + temporary_directory=temporary_directory, + buildbucket_id=buildbucket_id, + url=url, + ) + visited[buildbucket_id] = url + + exception_in_flight = False + except: + exception_in_flight = True + raise + finally: + if exception_in_flight: + # This is best-effort. If the machine powers off or similar, we'll just + # resubmit the same crashes, which is suboptimal, but otherwise + # acceptable. + logging.error( + "Something went wrong; attempting to save our work..." + ) + else: + logging.info("Persisting state...") + + tmp_state_file = state_file + ".tmp" + with open(tmp_state_file, "w", encoding="utf-8") as f: + json.dump(visited, f, indent=2) + os.rename(tmp_state_file, state_file) + + logging.info("State successfully persisted") + + if opts.cleanup: + shutil.rmtree(temporary_directory) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) |