diff options
author | android-build-team Robot <android-build-team-robot@google.com> | 2020-02-14 03:08:18 +0000 |
---|---|---|
committer | android-build-team Robot <android-build-team-robot@google.com> | 2020-02-14 03:08:18 +0000 |
commit | db729d703a0bf8fc92f011d160abd063c338b2f3 (patch) | |
tree | 35fa0fbaeaaddd9cc2a126a05eee3527b51e83a8 | |
parent | dddbf2c6f79f2334de6d41b86d63698714ba1124 (diff) | |
parent | b75f321fc8978b92ce3db6886ccb966768f0c7a8 (diff) | |
download | toolchain-utils-android11-gsi.tar.gz |
Snap for 6206568 from b75f321fc8978b92ce3db6886ccb966768f0c7a8 to rvc-releaseandroid-vts-11.0_r9android-vts-11.0_r8android-vts-11.0_r7android-vts-11.0_r6android-vts-11.0_r5android-vts-11.0_r4android-vts-11.0_r3android-vts-11.0_r2android-vts-11.0_r15android-vts-11.0_r14android-vts-11.0_r13android-vts-11.0_r12android-vts-11.0_r11android-vts-11.0_r10android-vts-11.0_r1android-security-11.0.0_r76android-security-11.0.0_r75android-security-11.0.0_r74android-security-11.0.0_r73android-security-11.0.0_r72android-security-11.0.0_r71android-security-11.0.0_r70android-security-11.0.0_r69android-security-11.0.0_r68android-security-11.0.0_r67android-security-11.0.0_r66android-security-11.0.0_r65android-security-11.0.0_r64android-security-11.0.0_r63android-security-11.0.0_r62android-security-11.0.0_r61android-security-11.0.0_r60android-security-11.0.0_r59android-security-11.0.0_r58android-security-11.0.0_r57android-security-11.0.0_r56android-security-11.0.0_r55android-security-11.0.0_r54android-security-11.0.0_r53android-security-11.0.0_r52android-security-11.0.0_r51android-security-11.0.0_r50android-security-11.0.0_r49android-security-11.0.0_r1android-platform-11.0.0_r9android-platform-11.0.0_r8android-platform-11.0.0_r7android-platform-11.0.0_r6android-platform-11.0.0_r5android-platform-11.0.0_r4android-platform-11.0.0_r3android-platform-11.0.0_r2android-platform-11.0.0_r13android-platform-11.0.0_r12android-platform-11.0.0_r11android-platform-11.0.0_r10android-platform-11.0.0_r1android-cts-11.0_r9android-cts-11.0_r8android-cts-11.0_r7android-cts-11.0_r6android-cts-11.0_r5android-cts-11.0_r4android-cts-11.0_r3android-cts-11.0_r2android-cts-11.0_r15android-cts-11.0_r14android-cts-11.0_r13android-cts-11.0_r12android-cts-11.0_r11android-cts-11.0_r10android-cts-11.0_r1android-11.0.0_r6android-11.0.0_r5android-11.0.0_r4android-11.0.0_r3android-11.0.0_r25android-11.0.0_r2android-11.0.0_r17android-11.0.0_r1android11-tests-releaseandroid11-security-releaseandroid11-s1-releaseandroid11-releaseandroid11-platform-releaseandroid11-gsi
Change-Id: I997cf91dd612b5e50d502271baf665232843c30a
217 files changed, 1545 insertions, 1213 deletions
diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 967c3486..751af04c 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -3,9 +3,9 @@ "name": "R81-12607.58-1578524353" }, "chromeos-kernel-4_4": { - "name": "R81-12861.0-1580726347" + "name": "R82-12874.0-1581330812" }, "chromeos-kernel-3_18": { - "name": "R81-12861.0-1580725953" + "name": "R82-12861.0-1580725956" } }
\ No newline at end of file diff --git a/afdo_redaction/redact_profile.py b/afdo_redaction/redact_profile.py index 96375fee..02bae928 100755 --- a/afdo_redaction/redact_profile.py +++ b/afdo_redaction/redact_profile.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -30,6 +30,7 @@ import collections import re import sys + def _count_samples(samples): """Count the total number of samples in a function.""" line_re = re.compile(r'^(\s*)\d+(?:\.\d+)?: (\d+)\s*$') @@ -132,7 +133,7 @@ def _read_textual_afdo_profile(stream): continue if line[0].isspace(): - assert function_line is not None, "sample exists outside of a function?" + assert function_line is not None, 'sample exists outside of a function?' samples.append(line) continue @@ -170,12 +171,13 @@ def dedup_records(profile_records, summary_file, max_repeats=100): counts[_normalize_samples(record.samples)].append(record) # Be sure that we didn't see any duplicate functions, since that's bad... - total_functions_recorded = sum(len(records) - for records in counts.itervalues()) + total_functions_recorded = sum(len(records) for records in counts.values()) - unique_function_names = set(record.function_line.split(':')[0] - for records in counts.itervalues() - for record in records) + unique_function_names = { + record.function_line.split(':')[0] + for records in counts.values() + for record in records + } assert len(unique_function_names) == total_functions_recorded, \ 'duplicate function names?' @@ -187,7 +189,7 @@ def dedup_records(profile_records, summary_file, max_repeats=100): num_samples_total = 0 num_top_samples_total = 0 - for normalized_samples, records in counts.iteritems(): + for normalized_samples, records in counts.items(): top_sample_count, all_sample_count = _count_samples(normalized_samples) top_sample_count *= len(records) all_sample_count *= len(records) @@ -205,11 +207,13 @@ def dedup_records(profile_records, summary_file, max_repeats=100): for record in records: yield record - print('Retained {:,}/{:,} functions'.format(num_kept, num_total), - file=summary_file) - print('Retained {:,}/{:,} samples, total'.format(num_samples_kept, - num_samples_total), - file=summary_file) + print( + 'Retained {:,}/{:,} functions'.format(num_kept, num_total), + file=summary_file) + print( + 'Retained {:,}/{:,} samples, total'.format(num_samples_kept, + num_samples_total), + file=summary_file) print('Retained {:,}/{:,} top-level samples' \ .format(num_top_samples_kept, num_top_samples_total), file=summary_file) @@ -220,15 +224,17 @@ def run(profile_input_file, summary_output_file, profile_output_file): # Sort this so we get deterministic output. AFDO doesn't care what order it's # in. - deduped = sorted(dedup_records(profile_records, summary_output_file), - key=lambda r: r.function_line) + deduped = sorted( + dedup_records(profile_records, summary_output_file), + key=lambda r: r.function_line) for function_line, samples in deduped: print(function_line, file=profile_output_file) print('\n'.join(samples), file=profile_output_file) def _main(): - run(profile_input_file=sys.stdin, summary_output_file=sys.stderr, + run(profile_input_file=sys.stdin, + summary_output_file=sys.stderr, profile_output_file=sys.stdout) diff --git a/afdo_redaction/redact_profile_test.py b/afdo_redaction/redact_profile_test.py index 27fb534e..e2438972 100755 --- a/afdo_redaction/redact_profile_test.py +++ b/afdo_redaction/redact_profile_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,12 +8,12 @@ from __future__ import division, print_function -import StringIO +import io import unittest -import redact_profile +from afdo_redaction import redact_profile -_redact_limit = redact_profile.dedup_records.func_defaults[0] +_redact_limit = redact_profile.dedup_records.__defaults__[0] def _redact(input_lines, summary_to=None): @@ -21,17 +21,18 @@ def _redact(input_lines, summary_to=None): input_lines = input_lines.splitlines() if summary_to is None: - summary_to = StringIO.StringIO() + summary_to = io.StringIO() - output_to = StringIO.StringIO() - redact_profile.run(profile_input_file=input_lines, - summary_output_file=summary_to, - profile_output_file=output_to) + output_to = io.StringIO() + redact_profile.run( + profile_input_file=input_lines, + summary_output_file=summary_to, + profile_output_file=output_to) return output_to.getvalue() def _redact_with_summary(input_lines): - summary = StringIO.StringIO() + summary = io.StringIO() result = _redact(input_lines, summary_to=summary) return result, summary.getvalue() @@ -64,6 +65,7 @@ def _generate_repeated_function_body(repeats, fn_name='_some_name'): class Tests(unittest.TestCase): """All of our tests for redact_profile.""" + def test_no_input_works(self): self.assertEqual(_redact(''), '') @@ -93,13 +95,13 @@ class Tests(unittest.TestCase): result_file = '\n'.join(kept_lines) + '\n' - lines = _generate_repeated_function_body(_redact_limit, - fn_name='_discard_me') + lines = _generate_repeated_function_body( + _redact_limit, fn_name='_discard_me') self.assertEqual(_redact(kept_lines + lines), result_file) self.assertEqual(_redact(lines + kept_lines), result_file) - more_lines = _generate_repeated_function_body(_redact_limit, - fn_name='_and_discard_me') + more_lines = _generate_repeated_function_body( + _redact_limit, fn_name='_and_discard_me') self.assertEqual(_redact(lines + kept_lines + more_lines), result_file) self.assertEqual(_redact(lines + more_lines), '') diff --git a/afdo_redaction/remove_indirect_calls.py b/afdo_redaction/remove_indirect_calls.py index b879b2f0..0dc15077 100755 --- a/afdo_redaction/remove_indirect_calls.py +++ b/afdo_redaction/remove_indirect_calls.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -21,7 +21,6 @@ from __future__ import division, print_function import argparse import re -import sys def _remove_indirect_call_targets(lines): diff --git a/afdo_redaction/remove_indirect_calls_test.py b/afdo_redaction/remove_indirect_calls_test.py index 1499af25..164b284f 100755 --- a/afdo_redaction/remove_indirect_calls_test.py +++ b/afdo_redaction/remove_indirect_calls_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -11,12 +11,12 @@ from __future__ import print_function import io import unittest -import remove_indirect_calls +from afdo_redaction import remove_indirect_calls def _run_test(input_lines): - input_buf = io.BytesIO('\n'.join(input_lines)) - output_buf = io.BytesIO() + input_buf = io.StringIO('\n'.join(input_lines)) + output_buf = io.StringIO() remove_indirect_calls.run(input_buf, output_buf) return output_buf.getvalue().splitlines() diff --git a/afdo_tools/bisection/afdo_prof_analysis.py b/afdo_tools/bisection/afdo_prof_analysis.py index 36531106..94e5366b 100755 --- a/afdo_tools/bisection/afdo_prof_analysis.py +++ b/afdo_tools/bisection/afdo_prof_analysis.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -109,7 +109,7 @@ class DeciderState(object): self.state_file) return - with open(self.state_file) as f: + with open(self.state_file, encoding='utf-8') as f: try: data = json.load(f) except: @@ -127,7 +127,7 @@ class DeciderState(object): def save_state(self): state = {'seed': self.seed, 'accumulated_results': self.accumulated_results} tmp_file = self.state_file + '.new' - with open(tmp_file, 'w') as f: + with open(tmp_file, 'w', encoding='utf-8') as f: json.dump(state, f, indent=2) os.rename(tmp_file, self.state_file) logging.info('Logged state to %s...', self.state_file) @@ -270,7 +270,7 @@ def range_search(decider, good, bad, common_funcs, lo, hi): def find_upper_border(good_copy, funcs, lo, hi, last_bad_val=None): """Finds the upper border of problematic range.""" mid = average(lo, hi) - if mid == lo or mid == hi: + if mid in (lo, hi): return last_bad_val or hi for func in funcs[lo:mid]: @@ -288,7 +288,7 @@ def range_search(decider, good, bad, common_funcs, lo, hi): def find_lower_border(good_copy, funcs, lo, hi, last_bad_val=None): """Finds the lower border of problematic range.""" mid = average(lo, hi) - if mid == lo or mid == hi: + if mid in (lo, hi): return last_bad_val or lo for func in funcs[lo:mid]: @@ -428,7 +428,7 @@ def main(flags): 'good_only_functions': gnb_result, 'bad_only_functions': bng_result } - with open(flags.analysis_output_file, 'wb') as f: + with open(flags.analysis_output_file, 'w', encoding='utf-8') as f: json.dump(results, f, indent=2) if flags.remove_state_on_completion: os.remove(flags.state_file) diff --git a/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py b/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py index 85c1c175..b293b8aa 100755 --- a/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py +++ b/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -15,13 +15,13 @@ import tempfile import unittest from datetime import date -import afdo_prof_analysis as analysis +from afdo_tools.bisection import afdo_prof_analysis as analysis class ObjectWithFields(object): """Turns kwargs given to the constructor into fields on an object. - Example usage: + Examples: x = ObjectWithFields(a=1, b=2) assert x.a == 1 assert x.b == 2 diff --git a/afdo_tools/bisection/afdo_prof_analysis_test.py b/afdo_tools/bisection/afdo_prof_analysis_test.py index 7bd3050c..245edc33 100755 --- a/afdo_tools/bisection/afdo_prof_analysis_test.py +++ b/afdo_tools/bisection/afdo_prof_analysis_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,12 +8,12 @@ from __future__ import print_function -import afdo_prof_analysis as analysis - import random -import StringIO +import io import unittest +from afdo_tools.bisection import afdo_prof_analysis as analysis + class AfdoProfAnalysisTest(unittest.TestCase): """Class for testing AFDO Profile Analysis""" @@ -33,17 +33,17 @@ class AfdoProfAnalysisTest(unittest.TestCase): analysis.random.seed(5) # 5 is an arbitrary choice. For consistent testing def test_text_to_json(self): - test_data = StringIO.StringIO('deflate_slow:87460059:3\n' - ' 3: 24\n' - ' 14: 54767\n' - ' 15: 664 fill_window:22\n' - ' 16: 661\n' - ' 19: 637\n' - ' 41: 36692 longest_match:36863\n' - ' 44: 36692\n' - ' 44.2: 5861\n' - ' 46: 13942\n' - ' 46.1: 14003\n') + test_data = io.StringIO('deflate_slow:87460059:3\n' + ' 3: 24\n' + ' 14: 54767\n' + ' 15: 664 fill_window:22\n' + ' 16: 661\n' + ' 19: 637\n' + ' 41: 36692 longest_match:36863\n' + ' 44: 36692\n' + ' 44.2: 5861\n' + ' 46: 13942\n' + ' 46.1: 14003\n') expected = { 'deflate_slow': ':87460059:3\n' ' 3: 24\n' @@ -115,7 +115,7 @@ class AfdoProfAnalysisTest(unittest.TestCase): self.bad_items, common_funcs, 0, len(common_funcs)) - self.assertEquals(['func_a', 'func_b'], problem_range) + self.assertEqual(['func_a', 'func_b'], problem_range) def test_check_good_not_bad(self): func_in_good = 'func_c' diff --git a/afdo_tools/generate_afdo_from_tryjob.py b/afdo_tools/generate_afdo_from_tryjob.py index b8a2d669..3ed578ea 100755 --- a/afdo_tools/generate_afdo_from_tryjob.py +++ b/afdo_tools/generate_afdo_from_tryjob.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -115,7 +115,7 @@ def main(): '--tryjob', required=True, type=_tryjob_arg, - help='Path to our tryjob\'s artifacts. Accepts a gs:// path, pantheon ' + help="Path to our tryjob's artifacts. Accepts a gs:// path, pantheon " 'link, or tryjob ID, e.g. R75-11965.0.0-b3648595. In the last case, ' 'the assumption is that you ran a chell-chrome-pfq-tryjob.') parser.add_argument( @@ -127,7 +127,7 @@ def main(): '-k', '--keep_artifacts_on_failure', action='store_true', - help='Don\'t remove the tempdir on failure') + help="Don't remove the tempdir on failure") args = parser.parse_args() if not distutils.spawn.find_executable(_CREATE_LLVM_PROF): diff --git a/afdo_tools/run_afdo_tryjob.py b/afdo_tools/run_afdo_tryjob.py index de45af0b..e14cd918 100755 --- a/afdo_tools/run_afdo_tryjob.py +++ b/afdo_tools/run_afdo_tryjob.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -94,7 +94,7 @@ def main(): user_patches = args.patch if tag_profiles_with_current_time and use_afdo_generation_stage: - raise ValueError('You can\'t tag profiles with the time + have ' + raise ValueError("You can't tag profiles with the time + have " 'afdo-generate') if not tag_profiles_with_current_time and not use_afdo_generation_stage: diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py index 699e12a0..429d0b4a 100755 --- a/auto_delete_nightly_test_data.py +++ b/auto_delete_nightly_test_data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2019 The Chromium OS Authors. All rights reserved. @@ -22,6 +22,8 @@ from cros_utils import constants from cros_utils import misc DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') +NIGHTLY_TESTS_WORKSPACE = os.path.join(constants.CROSTC_WORKSPACE, + 'nightly-tests') def CleanNumberedDir(s, dry_run=False): @@ -48,7 +50,7 @@ def CleanNumberedDir(s, dry_run=False): ## check 's' to make sure it is sane. A valid dir to be removed must be ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. valid_dir_pattern = ( - '^' + constants.CROSTC_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')') + '^' + NIGHTLY_TESTS_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')') if not re.search(valid_dir_pattern, s): print('Trying to delete an invalid dir "{0}" (must match "{1}"), ' 'please check.'.format(s, valid_dir_pattern)) @@ -196,7 +198,7 @@ def Main(argv): dated_dir = DIR_BY_WEEKDAY[i - 1] rv += 0 if CleanDatedDir( - os.path.join(constants.CROSTC_WORKSPACE, dated_dir), + os.path.join(NIGHTLY_TESTS_WORKSPACE, dated_dir), options.dry_run) else 1 diff --git a/binary_search_tool/MAINTENANCE b/binary_search_tool/MAINTENANCE index 8e5b3c61..8f96ff10 100644 --- a/binary_search_tool/MAINTENANCE +++ b/binary_search_tool/MAINTENANCE @@ -1,3 +1,7 @@ +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + This document is for future maintainers of the binary search/bisection tools. Authors: @@ -21,7 +25,9 @@ The following are good reference materials on how the tool works: TESTING: All unit tests live under the ./test directory. However, these tests -specifically test binary_search_state.py, binary_search_perforce.py, bisect.py. +specifically test binary_search_state.py, binary_search_perforce.py, +run_bisect.py. + These unit tests will not test the specific logic for ChromeOS/Android bisection. To test the ChromeOS/Android bisectors, use the common/hash_test.sh test. This is a simple test case that just checks the hashes of files on your @@ -62,10 +68,11 @@ Some of the design decisions are a bit difficult to understand from just reading the code unfortunately. I will attempt to clear up the major offenders of this: 1. common.py's argument dictionary: - binary_search_state.py and bisect.py both have to have near identical - arguments in order to support argument overriding in bisect.py. However - they do have to be slightly different. Mainly, bisect.py needs to have no - default values for arguments (so it can determine what's being overriden). + binary_search_state.py and run_bisect.py both have to have near identical + arguments in order to support argument overriding in run_bisect.py. However + they do have to be slightly different. Mainly, run_bisect.py needs to have + no default values for arguments (so it can determine what's being + overriden). In order to reduce huge amounts of code duplication for the argument building, we put argument building in common.py. That way both modules diff --git a/binary_search_tool/README.bisect.md b/binary_search_tool/README.bisect.md index 74715ca0..bd9e0f14 100644 --- a/binary_search_tool/README.bisect.md +++ b/binary_search_tool/README.bisect.md @@ -1,6 +1,6 @@ -# `bisect.py` +# `run_bisect.py` -`bisect.py` is a wrapper around the general purpose +`run_bisect.py` is a wrapper around the general purpose `binary_search_state.py`. It provides a user friendly interface for bisecting various compilation errors. The 2 currently provided methods of bisecting are ChromeOS package and object bisection. Each @@ -33,8 +33,8 @@ pings the machine and prompts the user if the machine is good. /build/${board}.work - A full copy of /build/${board}.bad ``` -1. Cleanup: bisect.py does most cleanup for you, the only thing required by the - user is to cleanup all built images and the three build trees made in +1. Cleanup: run_bisect.py does most cleanup for you, the only thing required by + the user is to cleanup all built images and the three build trees made in `/build/` 1. Default Arguments: @@ -58,13 +58,13 @@ pings the machine and prompts the user if the machine is good. 1. Basic interactive test package bisection, on daisy board: ``` - ./bisect.py package daisy 172.17.211.184 + ./run_bisect.py package daisy 172.17.211.184 ``` 2. Basic boot test package bisection, on daisy board: ``` - ./bisect.py package daisy 172.17.211.184 -t cros_pkg/boot_test.sh + ./run_bisect.py package daisy 172.17.211.184 -t cros_pkg/boot_test.sh ``` ### ChromeOS Object @@ -125,10 +125,10 @@ pings the machine and prompts the user if the machine is good. 1. Examples: 1. Basic interactive test object bisection, on daisy board for cryptohome - package: `./bisect.py object daisy 172.17.211.184 cryptohome` + package: `./run_bisect.py object daisy 172.17.211.184 cryptohome` 2. Basic boot test package bisection, on daisy board for cryptohome - package: `./bisect.py object daisy 172.17.211.184 cryptohome + package: `./run_bisect.py object daisy 172.17.211.184 cryptohome --test_script=sysroot_wrapper/boot_test.sh` ### Android object @@ -202,16 +202,17 @@ the following arguments: 1. Examples: 1. Basic interactive test android bisection, where the android source is at - ~/android_src: `./bisect.py android ~/android_src` + ~/android_src: `./run_bisect.py android ~/android_src` 2. Basic boot test android bisection, where the android source is at - `~/android_src`, and 10 jobs will be used to build android: `./bisect.py + `~/android_src`, and 10 jobs will be used to build android: + `./run_bisect.py android ~/android_src --num_jobs=10 --test_script=sysroot_wrapper/boot_test.sh` ### Resuming -`bisect.py` and `binary_search_state.py` offer the +`run_bisect.py` and `binary_search_state.py` offer the ability to resume a bisection in case it was interrupted by a SIGINT, power failure, etc. Every time the tool completes a bisection iteration its state is saved to disk (usually to the file @@ -221,14 +222,14 @@ completed iteration. ### Overriding -You can run `./bisect.py --help` or `./binary_search_state.py +You can run `./run_bisect.py --help` or `./binary_search_state.py --help` for a full list of arguments that can be overriden. Here are a couple of examples: Example 1 (do boot test instead of interactive test): ``` -./bisect.py package daisy 172.17.211.182 --test_script=cros_pkg/boot_test.sh +./run_bisect.py package daisy 172.17.211.182 --test_script=cros_pkg/boot_test.sh ``` Example 2 (do package bisector system test instead of interactive test, this @@ -236,13 +237,13 @@ Example 2 (do package bisector system test instead of interactive test, this hash_test.sh for more details): ``` -./bisect.py package daisy 172.17.211.182 \ +./run_bisect.py package daisy 172.17.211.182 \ --test_script=common/hash_test.sh --test_setup_script="" ``` Example 3 (enable verbose mode, disable pruning, and disable verification): ``` -./bisect.py package daisy 172.17.211.182 +./run_bisect.py package daisy 172.17.211.182 --verbose --prune=False --verify=False ``` diff --git a/binary_search_tool/README.pass_bisect.md b/binary_search_tool/README.pass_bisect.md index d03a563b..eb0f12a6 100644 --- a/binary_search_tool/README.pass_bisect.md +++ b/binary_search_tool/README.pass_bisect.md @@ -41,7 +41,7 @@ them. bisection: ``` - ./bisect.py android PATH_TO_ANDROID_HOME_DIR + ./run_bisect.py android PATH_TO_ANDROID_HOME_DIR --pass_bisect=’android/generate_cmd.sh’ --prune=False --ir_diff diff --git a/binary_search_tool/README.testing.md b/binary_search_tool/README.testing.md index 139dc087..0f06679b 100644 --- a/binary_search_tool/README.testing.md +++ b/binary_search_tool/README.testing.md @@ -28,7 +28,7 @@ To run the basic unit tests: ``` $ cd toolchain-utils/binary_search_tool/test -$ ./binary_search_tool_tester.py +$ ./binary_search_tool_test.py ``` # Running the bisection tests, testing the compiler wrapper. @@ -59,7 +59,7 @@ Move to the correct directory, then run the test script: ``` $ cd toolchain-utils/binary_search_tool -$ ./run_bisect_test.py +$ ./run_bisect_tests.py ``` If you are testing with the CHROMEOS COMPILER WRAPPER, you MUST run the @@ -82,7 +82,7 @@ $ export PYTHONPATH=`pwd`:${PYTHONPATH} Run the test script: ``` -$ ./run_bisect_test.py +$ ./run_bisect_tests.py ``` diff --git a/binary_search_tool/__init__.py b/binary_search_tool/__init__.py index 8b137891..76500def 100644 --- a/binary_search_tool/__init__.py +++ b/binary_search_tool/__init__.py @@ -1 +1,4 @@ - +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/binary_search_tool/binary_search_perforce.py b/binary_search_tool/binary_search_perforce.py index a4f8c1c6..5065f709 100755 --- a/binary_search_tool/binary_search_perforce.py +++ b/binary_search_tool/binary_search_perforce.py @@ -1,9 +1,11 @@ -#!/usr/bin/env python2 - -# Copyright 2018 The Chromium OS Authors. All rights reserved. +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Module of binary serch for perforce.""" +from __future__ import division from __future__ import print_function import math @@ -25,7 +27,7 @@ def _GetP4ClientSpec(client_name, p4_paths): if ' ' not in p4_path: p4_string += ' -a %s' % p4_path else: - p4_string += " -a \"" + (' //' + client_name + '/').join(p4_path) + "\"" + p4_string += ' -a "' + (' //' + client_name + '/').join(p4_path) + '"' return p4_string @@ -50,8 +52,8 @@ def GetP4Command(client_name, p4_port, p4_paths, checkoutdir, p4_snapshot=''): command += ' && cd ' + checkoutdir command += ' && cp ${HOME}/.p4config .' command += ' && chmod u+w .p4config' - command += " && echo \"P4PORT=" + p4_port + "\" >> .p4config" - command += " && echo \"P4CLIENT=" + client_name + "\" >> .p4config" + command += ' && echo "P4PORT=' + p4_port + '" >> .p4config' + command += ' && echo "P4CLIENT=' + client_name + '" >> .p4config' command += (' && g4 client ' + _GetP4ClientSpec(client_name, p4_paths)) command += ' && g4 sync ' command += ' && cd -' @@ -84,7 +86,7 @@ class BinarySearcherForPass(object): # For the first run, update self.hi with total pass/transformation count if self.hi == 0: self.hi = self.total - self.current = (self.hi + self.lo) / 2 + self.current = (self.hi + self.lo) // 2 message = ('Bisecting between: (%d, %d)' % (self.lo, self.hi)) self.logger.LogOutput(message, print_to_console=verbose) message = ('Current limit number: %d' % self.current) @@ -100,11 +102,11 @@ class BinarySearcherForPass(object): If status == 1, it means that runtime error still happens with current pass/ transformation, so we need to decrease upper bound for binary search. - Return: + Returns: True if we find the bad pass/transformation, or cannot find bad one after decreasing to the first pass/transformation. Otherwise False. """ - assert status == 0 or status == 1 or status == 125 + assert status in (0, 1, 125), status if self.current == 0: message = ('Runtime error occurs before first pass/transformation. ' @@ -145,7 +147,7 @@ class BinarySearcher(object): self.logger = logger.GetLogger() def SetSortedList(self, sorted_list): - assert len(sorted_list) > 0 + assert sorted_list self.sorted_list = sorted_list self.index_log = [] self.hi = len(sorted_list) - 1 @@ -159,7 +161,7 @@ class BinarySearcher(object): message = ('Revision: %s index: %d returned: %d' % (self.sorted_list[self.current], self.current, status)) self.logger.LogOutput(message, print_to_console=verbose) - assert status == 0 or status == 1 or status == 125 + assert status in (0, 1, 125), status self.index_log.append(self.current) self.status_log.append(status) bsp = BinarySearchPoint(self.sorted_list[self.current], status, tag) @@ -168,13 +170,13 @@ class BinarySearcher(object): if status == 125: self.skipped_indices.append(self.current) - if status == 0 or status == 1: + if status in (0, 1): if status == 0: self.lo = self.current + 1 elif status == 1: self.hi = self.current self.logger.LogOutput('lo: %d hi: %d\n' % (self.lo, self.hi)) - self.current = (self.lo + self.hi) / 2 + self.current = (self.lo + self.hi) // 2 if self.lo == self.hi: message = ('Search complete. First bad version: %s' @@ -194,19 +196,19 @@ class BinarySearcher(object): def GetNextFlakyBinary(self): t = (self.lo, self.current, self.hi) q = [t] - while len(q): + while q: element = q.pop(0) if element[1] in self.skipped_indices: # Go top - to_add = (element[0], (element[0] + element[1]) / 2, element[1]) + to_add = (element[0], (element[0] + element[1]) // 2, element[1]) q.append(to_add) # Go bottom - to_add = (element[1], (element[1] + element[2]) / 2, element[2]) + to_add = (element[1], (element[1] + element[2]) // 2, element[2]) q.append(to_add) else: self.current = element[1] return - assert len(q), 'Queue should never be 0-size!' + assert q, 'Queue should never be 0-size!' def GetNextFlakyLinear(self): current_hi = self.current @@ -225,7 +227,7 @@ class BinarySearcher(object): current_lo -= 1 def GetNext(self): - self.current = (self.hi + self.lo) / 2 + self.current = (self.hi + self.lo) // 2 # Try going forward if current is skipped. if self.current in self.skipped_indices: self.GetNextFlakyBinary() @@ -295,7 +297,7 @@ class VCSBinarySearcher(object): def GetNextRevision(self): pass - def CheckoutRevision(self, revision): + def CheckoutRevision(self, current_revision): pass def SetStatus(self, status): @@ -362,14 +364,14 @@ class P4BinarySearcher(VCSBinarySearcher): if not os.path.isfile(self.checkout_dir + '/.p4config'): command = 'cd %s' % self.checkout_dir command += ' && cp ${HOME}/.p4config .' - command += " && echo \"P4PORT=" + self.p4_port + "\" >> .p4config" - command += " && echo \"P4CLIENT=" + self.client_name + "\" >> .p4config" + command += ' && echo "P4PORT=' + self.p4_port + '" >> .p4config' + command += ' && echo "P4CLIENT=' + self.client_name + '" >> .p4config' self.ce.RunCommand(command) command = 'cd %s' % self.checkout_dir command += '; g4 changes -c %s' % self.client_name _, out, _ = self.ce.RunCommandWOUTPUOT(command) changes = re.findall(r'Change (\d+)', out) - if len(changes) != 0: + if changes: command = 'cd %s' % self.checkout_dir for change in changes: command += '; g4 revert -c %s' % change @@ -413,9 +415,8 @@ class P4GCCBinarySearcher(P4BinarySearcher): for pr in problematic_ranges: if cr in range(pr[0], pr[1]): patch_file = '/home/asharif/triage_tool/%d-%d.patch' % (pr[0], pr[1]) - f = open(patch_file) - patch = f.read() - f.close() + with open(patch_file, encoding='utf-8') as f: + patch = f.read() files = re.findall('--- (//.*)', patch) command += '; cd %s' % self.checkout_dir for f in files: @@ -505,7 +506,7 @@ def Main(argv): logger.GetLogger().LogOutput('Cleaning up...') finally: logger.GetLogger().LogOutput(str(p4gccbs.bs), print_to_console=verbose) - status = p4gccbs.Cleanup() + p4gccbs.Cleanup() if __name__ == '__main__': diff --git a/binary_search_tool/binary_search_state.py b/binary_search_tool/binary_search_state.py index f6c8ac7c..1ddd65ce 100755 --- a/binary_search_tool/binary_search_state.py +++ b/binary_search_tool/binary_search_state.py @@ -1,10 +1,12 @@ -#!/usr/bin/env python2 - -# Copyright 2018 The Chromium OS Authors. All rights reserved. +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """The binary search wrapper.""" +from __future__ import division from __future__ import print_function import argparse @@ -20,15 +22,14 @@ import tempfile import time # Adds cros_utils to PYTHONPATH -import common +from binary_search_tool import binary_search_perforce +from binary_search_tool import common +from binary_search_tool import pass_mapping # Now we do import from cros_utils from cros_utils import command_executer from cros_utils import logger -import binary_search_perforce -import pass_mapping - GOOD_SET_VAR = 'BISECT_GOOD_SET' BAD_SET_VAR = 'BISECT_BAD_SET' @@ -37,11 +38,6 @@ HIDDEN_STATE_FILE = os.path.join( os.path.dirname(STATE_FILE), '.%s' % os.path.basename(STATE_FILE)) -class Error(Exception): - """The general binary search tool error class.""" - pass - - @contextlib.contextmanager def SetFile(env_var, items): """Generate set files that can be used by switch/test scripts. @@ -59,7 +55,7 @@ def SetFile(env_var, items): env_var: What environment variable to store the file name in. items: What items are in this set. """ - with tempfile.NamedTemporaryFile() as f: + with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f: os.environ[env_var] = f.name f.write('\n'.join(items)) f.flush() @@ -100,10 +96,10 @@ class BinarySearchState(object): self.cmd_script = None self.mode = None self.PopulateItemsUsingCommand(self.get_initial_items) - self.currently_good_items = set([]) - self.currently_bad_items = set([]) - self.found_items = set([]) - self.known_good = set([]) + self.currently_good_items = set() + self.currently_bad_items = set() + self.found_items = set() + self.known_good = set() self.start_time = time.time() @@ -161,7 +157,7 @@ class BinarySearchState(object): item_list: list of all items to be switched """ if self.file_args: - with tempfile.NamedTemporaryFile() as f: + with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f: f.write('\n'.join(item_list)) f.flush() command = '%s %s' % (switch_script, f.name) @@ -174,9 +170,8 @@ class BinarySearchState(object): command, print_to_console=self.verbose) except OSError as e: if e.errno == errno.E2BIG: - raise Error('Too many arguments for switch script! Use --file_args') - else: - raise + raise RuntimeError('Too many arguments for switch script! Use ' + '--file_args') assert ret == 0, 'Switch script %s returned %d' % (switch_script, ret) def TestScript(self): @@ -349,12 +344,12 @@ class BinarySearchState(object): return None def BuildWithPassLimit(self, limit, generate_ir=False): - """ Rebuild bad item with pass level bisect limit + """Rebuild bad item with pass level bisect limit Run command line script generated by GenerateBadCommandScript(), with pass level limit flags. - Return: + Returns: pass_num: current number of the pass, or total number of passes if limit set to -1. pass_name: The debugcounter name of current limit pass. @@ -389,7 +384,7 @@ class BinarySearchState(object): break pass_num += 1 last_pass = l - if limit != -1 and pass_num != limit: + if limit not in (-1, pass_num): raise ValueError('[Error] While building, limit number does not match.') return pass_num, self.CollectPassName(last_pass) @@ -398,16 +393,19 @@ class BinarySearchState(object): pass_name=None, pass_limit=-1, generate_ir=False): - """ Rebuild bad item with transformation level bisect limit + """Rebuild bad item with transformation level bisect limit Run command line script generated by GenerateBadCommandScript(), with pass level limit flags and transformation level limit flags. Args: - limit: transformation level limit for bad item - pass_name: name of bad pass debugcounter from pass level bisect result - pass_limit: pass level limit from pass level bisect result - Return: Total number of transformations if limit set to -1, else return 0. + limit: transformation level limit for bad item. + pass_name: name of bad pass debugcounter from pass level bisect result. + pass_limit: pass level limit from pass level bisect result. + generate_ir: Whether to generate IR comparison. + + Returns: + Total number of transformations if limit set to -1, else return 0. """ counter_name = pass_name @@ -513,7 +511,7 @@ class BinarySearchState(object): 'Total %s number: %d' % (self.mode, self.binary_search.total)) trans_index, _ = self.DoBinarySearchBadPass(pass_index, pass_name) - if (trans_index == 0): + if trans_index == 0: raise ValueError('Bisecting %s cannot reproduce good result.' % pass_name) if self.ir_diff: @@ -530,13 +528,15 @@ class BinarySearchState(object): Args: pass_index: Works for transformation level bisection, indicates the limit - number of pass from pass level bisecting result. + number of pass from pass level bisecting result. pass_name: Works for transformation level bisection, indicates - DebugCounter name of the bad pass from pass level bisecting result. - Return: + DebugCounter name of the bad pass from pass level bisecting + result. + + Returns: index: Index of problematic pass/transformation. pass_name: Works for pass level bisection, returns DebugCounter name for - bad pass. + bad pass. """ # If in resume mode don't reset search_cycles if not self.resumed: @@ -609,7 +609,7 @@ class BinarySearchState(object): new data. Raises: - Error if STATE_FILE already exists but is not a symlink. + OSError if STATE_FILE already exists but is not a symlink. """ ce, l = self.ce, self.l self.ce, self.l, self.binary_search.logger = None, None, None @@ -623,8 +623,8 @@ class BinarySearchState(object): if os.path.islink(STATE_FILE): old_state = os.readlink(STATE_FILE) else: - raise Error(('%s already exists and is not a symlink!\n' - 'State file saved to %s' % (STATE_FILE, path))) + raise OSError(('%s already exists and is not a symlink!\n' + 'State file saved to %s' % (STATE_FILE, path))) # Create new link and atomically overwrite old link temp_link = '%s.link' % HIDDEN_STATE_FILE @@ -642,26 +642,27 @@ class BinarySearchState(object): if not os.path.isfile(STATE_FILE): return None try: - bss = pickle.load(file(STATE_FILE)) - bss.l = logger.GetLogger() - bss.ce = command_executer.GetCommandExecuter() - bss.binary_search.logger = bss.l - bss.start_time = time.time() - - # Set resumed to be True so we can enter DoBinarySearch without the method - # resetting our current search_cycles to 0. - bss.resumed = True - - # Set currently_good_items and currently_bad_items to empty so that the - # first iteration after resuming will always be non-incremental. This is - # just in case the environment changes, the user makes manual changes, or - # a previous switch_script corrupted the environment. - bss.currently_good_items = set([]) - bss.currently_bad_items = set([]) - - binary_search_perforce.verbose = bss.verbose - return bss - except StandardError: + with open(STATE_FILE, 'rb') as f: + bss = pickle.load(f) + bss.l = logger.GetLogger() + bss.ce = command_executer.GetCommandExecuter() + bss.binary_search.logger = bss.l + bss.start_time = time.time() + + # Set resumed to be True so we can enter DoBinarySearch without the + # method resetting our current search_cycles to 0. + bss.resumed = True + + # Set currently_good_items and currently_bad_items to empty so that the + # first iteration after resuming will always be non-incremental. This + # is just in case the environment changes, the user makes manual + # changes, or a previous switch_script corrupted the environment. + bss.currently_good_items = set() + bss.currently_bad_items = set() + + binary_search_perforce.verbose = bss.verbose + return bss + except Exception: return None def RemoveState(self): @@ -686,8 +687,8 @@ class BinarySearchState(object): """Return h m s format of elapsed time since execution has started.""" diff = int(time.time() - self.start_time) seconds = diff % 60 - minutes = (diff / 60) % 60 - hours = diff / (60 * 60) + minutes = (diff // 60) % 60 + hours = diff // (60 * 60) seconds = str(seconds).rjust(2) minutes = str(minutes).rjust(2) @@ -871,16 +872,12 @@ def Run(get_initial_items, verify, file_args, verbose) bss.DoVerify() - try: - bss.DoSearchBadItems() - if pass_bisect: - bss.DoSearchBadPass() - bss.RemoveState() - logger.GetLogger().LogOutput( - 'Total execution time: %s' % bss.ElapsedTimeString()) - except Error as e: - logger.GetLogger().LogError(e) - return 1 + bss.DoSearchBadItems() + if pass_bisect: + bss.DoSearchBadPass() + bss.RemoveState() + logger.GetLogger().LogOutput( + 'Total execution time: %s' % bss.ElapsedTimeString()) return 0 diff --git a/binary_search_tool/bisect_driver.py b/binary_search_tool/bisect_driver.py index 82df15d6..ac37ad9f 100644 --- a/binary_search_tool/bisect_driver.py +++ b/binary_search_tool/bisect_driver.py @@ -1,10 +1,10 @@ -# Copyright 2016 Googie Inc. All rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. # # This script is used to help the compiler wrapper in the ChromeOS and # Android build systems bisect for bad object files. -# -# pylint: disable=not-callable -# pylint: disable=indentation """Utilities for bisection of ChromeOS and Android object files. @@ -267,10 +267,10 @@ def cache_file(execargs, bisect_dir, cache, abs_file_path): if CONTINUE_ON_REDUNDANCY: return True raise Exception( - 'Trying to cache file %s multiple times. To avoid the error, set \ - CONTINUE_ON_REDUNDANCY to 1. For reference, the list of such files \ - will be written to %s' % (abs_file_path, - os.path.join(population_dir, '_DUPS'))) + 'Trying to cache file %s multiple times. To avoid the error, set ' \ + 'BISECT_CONTINUE_ON_REDUNDANCY to 1. For reference, the list of ' \ + 'such files will be written to %s' % (abs_file_path, os.path.join( + population_dir, '_DUPS'))) shutil.copy2(abs_file_path, bisect_path) # Set cache object to be read-only so later compilations can't @@ -278,7 +278,8 @@ def cache_file(execargs, bisect_dir, cache, abs_file_path): os.chmod(bisect_path, 0o444) return True else: - # File not found (happens when compilation fails but error code is still 0) + # File not found (happens when compilation fails but error code is still + # 0) return False except Exception: print('Could not cache file %s' % abs_file_path, file=sys.stderr) diff --git a/binary_search_tool/common.py b/binary_search_tool/common.py index 40660b52..85cd478b 100644 --- a/binary_search_tool/common.py +++ b/binary_search_tool/common.py @@ -1,4 +1,5 @@ -# Copyright 2018 The Chromium OS Authors. All rights reserved. +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,14 +8,14 @@ This module serves two main purposes: 1. Programatically include the utils module in PYTHONPATH 2. Create the argument parsing shared between binary_search_state.py and - bisect.py + run_bisect.py The argument parsing is handled by populating _ArgsDict with all arguments. -_ArgsDict is required so that binary_search_state.py and bisect.py can share -the argument parsing, but treat them slightly differently. For example, -bisect.py requires that all argument defaults are suppressed so that overriding -can occur properly (i.e. only options that are explicitly entered by the user -end up in the resultant options dictionary). +_ArgsDict is required so that binary_search_state.py and run_bisect.py can +share the argument parsing, but treat them slightly differently. For example, +run_bisect.py requires that all argument defaults are suppressed so that +overriding can occur properly (i.e. only options that are explicitly entered +by the user end up in the resultant options dictionary). ArgumentDict inherits OrderedDict in order to preserve the order the args are created so the help text is made properly. @@ -32,8 +33,8 @@ if os.path.isabs(sys.argv[0]): os.path.dirname(sys.argv[0]))) else: wdir = os.getcwd() - utils_pythonpath = os.path.abspath('{0}/{1}/..'.format(wdir, os.path.dirname( - sys.argv[0]))) + utils_pythonpath = os.path.abspath('{0}/{1}/..'.format( + wdir, os.path.dirname(sys.argv[0]))) sys.path.append(utils_pythonpath) @@ -55,8 +56,10 @@ class ArgumentDict(collections.OrderedDict): ... } """ - _POSSIBLE_OPTIONS = ['action', 'nargs', 'const', 'default', 'type', 'choices', - 'required', 'help', 'metavar', 'dest'] + _POSSIBLE_OPTIONS = [ + 'action', 'nargs', 'const', 'default', 'type', 'choices', 'required', + 'help', 'metavar', 'dest' + ] def AddArgument(self, *args, **kwargs): """Add argument to ArgsDict, has same signature as argparse.add_argument @@ -77,7 +80,7 @@ class ArgumentDict(collections.OrderedDict): TypeError: if args is empty or if option in kwargs is not a valid option for argparse.add_argument. """ - if len(args) == 0: + if not args: raise TypeError('Argument needs at least one name') for key in kwargs: @@ -106,8 +109,8 @@ def BuildArgParser(parser, override=False): Args: parser: type argparse.ArgumentParser, will call add_argument for every item in _ArgsDict - override: True if being called from bisect.py. Used to say that default and - required options are to be ignored + override: True if being called from run_bisect.py. Used to say that default + and required options are to be ignored Returns: None @@ -115,7 +118,7 @@ def BuildArgParser(parser, override=False): ArgsDict = GetArgsDict() # Have no defaults when overriding - for arg_names, arg_options in ArgsDict.iteritems(): + for arg_names, arg_options in ArgsDict.items(): if override: arg_options = arg_options.copy() arg_options.pop('default', None) @@ -147,34 +150,34 @@ def _BuildArgsDict(args): '--get_initial_items', dest='get_initial_items', help='Script to run to get the initial objects. ' - 'If your script requires user input ' - 'the --verbose option must be used') + 'If your script requires user input ' + 'the --verbose option must be used') args.AddArgument( '-g', '--switch_to_good', dest='switch_to_good', help='Script to run to switch to good. ' - 'If your switch script requires user input ' - 'the --verbose option must be used') + 'If your switch script requires user input ' + 'the --verbose option must be used') args.AddArgument( '-b', '--switch_to_bad', dest='switch_to_bad', help='Script to run to switch to bad. ' - 'If your switch script requires user input ' - 'the --verbose option must be used') + 'If your switch script requires user input ' + 'the --verbose option must be used') args.AddArgument( '-I', '--test_setup_script', dest='test_setup_script', help='Optional script to perform building, flashing, ' - 'and other setup before the test script runs.') + 'and other setup before the test script runs.') args.AddArgument( '-t', '--test_script', dest='test_script', help='Script to run to test the ' - 'output after packages are built.') + 'output after packages are built.') # No input (evals to False), # --prune (evals to True), # --prune=False, @@ -189,19 +192,19 @@ def _BuildArgsDict(args): type=StrToBool, metavar='bool', help='If True, continue until all bad items are found. ' - 'Defaults to False.') + 'Defaults to False.') args.AddArgument( '-P', '--pass_bisect', dest='pass_bisect', default=None, help='Script to generate another script for pass level bisect, ' - 'which contains command line options to build bad item. ' - 'This will also turn on pass/transformation level bisection. ' - 'Needs support of `-opt-bisect-limit`(pass) and ' - '`-print-debug-counter`(transformation) from LLVM. ' - 'For now it only supports one single bad item, so to use it, ' - 'prune must be set to False.') + 'which contains command line options to build bad item. ' + 'This will also turn on pass/transformation level bisection. ' + 'Needs support of `-opt-bisect-limit`(pass) and ' + '`-print-debug-counter`(transformation) from LLVM. ' + 'For now it only supports one single bad item, so to use it, ' + 'prune must be set to False.') # No input (evals to False), # --ir_diff (evals to True), # --ir_diff=False, @@ -216,8 +219,8 @@ def _BuildArgsDict(args): type=StrToBool, metavar='bool', help='Whether to print IR differences before and after bad ' - 'pass/transformation to verbose output. Defaults to False, ' - 'only works when pass_bisect is enabled.') + 'pass/transformation to verbose output. Defaults to False, ' + 'only works when pass_bisect is enabled.') # No input (evals to False), # --noincremental (evals to True), # --noincremental=False, @@ -231,8 +234,8 @@ def _BuildArgsDict(args): default=False, type=StrToBool, metavar='bool', - help='If True, don\'t propagate good/bad changes ' - 'incrementally. Defaults to False.') + help="If True, don't propagate good/bad changes " + 'incrementally. Defaults to False.') # No input (evals to False), # --file_args (evals to True), # --file_args=False, @@ -247,7 +250,7 @@ def _BuildArgsDict(args): type=StrToBool, metavar='bool', help='Whether to use a file to pass arguments to scripts. ' - 'Defaults to False.') + 'Defaults to False.') # No input (evals to True), # --verify (evals to True), # --verify=False, @@ -261,7 +264,7 @@ def _BuildArgsDict(args): type=StrToBool, metavar='bool', help='Whether to run verify iterations before searching. ' - 'Defaults to True.') + 'Defaults to True.') args.AddArgument( '-N', '--prune_iterations', @@ -289,5 +292,5 @@ def _BuildArgsDict(args): dest='resume', action='store_true', help='Resume bisection tool execution from state file.' - 'Useful if the last bisection was terminated ' - 'before it could properly finish.') + 'Useful if the last bisection was terminated ' + 'before it could properly finish.') diff --git a/binary_search_tool/compiler_wrapper.py b/binary_search_tool/compiler_wrapper.py index a6d189b5..8755ed42 100755 --- a/binary_search_tool/compiler_wrapper.py +++ b/binary_search_tool/compiler_wrapper.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Prototype compiler wrapper. Only tested with: gcc, g++, clang, clang++ @@ -21,7 +26,7 @@ import os import shlex import sys -import bisect_driver +from binary_search_tool import bisect_driver WRAPPED = '%s.real' % sys.argv[0] BISECT_STAGE = os.environ.get('BISECT_STAGE') diff --git a/binary_search_tool/cros_pkg/create_cleanup_script.py b/binary_search_tool/cros_pkg/create_cleanup_script.py index ed4eab61..62ee38f1 100755 --- a/binary_search_tool/cros_pkg/create_cleanup_script.py +++ b/binary_search_tool/cros_pkg/create_cleanup_script.py @@ -1,6 +1,9 @@ -#!/usr/bin/env python2 -# -# Copyright 2015 Google Inc. All Rights Reserved +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """The script to generate a cleanup script after setup.sh. This script takes a set of flags, telling it what setup.sh changed @@ -69,20 +72,22 @@ def Main(argv): if options.old_link or options.renamed_tree: if not options.tree_existed: - Usage(parser, 'If --tree_existed is False, cannot have ' - '--renamed_tree or --old_link') + Usage( + parser, 'If --tree_existed is False, cannot have ' + '--renamed_tree or --old_link') if options.old_link and options.renamed_tree: Usage(parser, '--old_link and --renamed_tree are incompatible options.') if options.tree_existed: if not options.old_link and not options.renamed_tree: - Usage(parser, 'If --tree_existed is True, then must have either ' - '--old_link or --renamed_tree') + Usage( + parser, 'If --tree_existed is True, then must have either ' + '--old_link or --renamed_tree') out_filename = 'cros_pkg/' + options.board + '_cleanup.sh' - with open(out_filename, 'w') as out_file: + with open(out_filename, 'w', encoding='utf-8') as out_file: out_file.write('#!/bin/bash\n\n') # First, remove the 'new' soft link. out_file.write('sudo rm /build/%s\n' % options.board) @@ -100,8 +105,8 @@ def Main(argv): original_link = original_link[1:] if original_link[-1] == "'": original_link = original_link[:-1] - out_file.write('sudo ln -s %s /build/%s\n' % (original_link, - options.board)) + out_file.write( + 'sudo ln -s %s /build/%s\n' % (original_link, options.board)) out_file.write('\n') # Remove common.sh file out_file.write('rm common/common.sh\n') diff --git a/binary_search_tool/pass_mapping.py b/binary_search_tool/pass_mapping.py index cb80910c..2678fd6d 100644 --- a/binary_search_tool/pass_mapping.py +++ b/binary_search_tool/pass_mapping.py @@ -1,6 +1,8 @@ -# Copyright 2018 The Chromium OS Authors. All rights reserved. +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Config file for pass level bisection Provides a mapping from pass info from -opt-bisect result to DebugCounter name. diff --git a/binary_search_tool/bisect.py b/binary_search_tool/run_bisect.py index c7dd5238..ef1048bb 100755 --- a/binary_search_tool/bisect.py +++ b/binary_search_tool/run_bisect.py @@ -1,28 +1,29 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """The unified package/object bisecting tool.""" from __future__ import print_function import abc import argparse +from argparse import RawTextHelpFormatter import os import sys -from argparse import RawTextHelpFormatter -import common +from binary_search_tool import binary_search_state +from binary_search_tool import common from cros_utils import command_executer from cros_utils import logger -import binary_search_state - -class Bisector(object): +class Bisector(object, metaclass=abc.ABCMeta): """The abstract base class for Bisectors.""" - # Make Bisector an abstract class - __metaclass__ = abc.ABCMeta - def __init__(self, options, overrides=None): """Constructor for Bisector abstract base class @@ -44,7 +45,7 @@ class Bisector(object): which arguments have been overridden. Example output: - ./bisect.py package daisy 172.17.211.184 -I "" -t cros_pkg/my_test.sh + ./run_bisect.py package daisy 172.17.211.184 -I "" -t cros_pkg/my_test.sh Performing ChromeOS Package bisection Method Config: board : daisy @@ -83,10 +84,8 @@ class Bisector(object): out += '\nBisection Config: (* = overridden)\n' max_key_len = max([len(str(x)) for x in args.keys()]) # Print args in common._ArgsDict order - args_order = [x['dest'] for x in common.GetArgsDict().itervalues()] - compare = lambda x, y: cmp(args_order.index(x), args_order.index(y)) - - for key in sorted(args, cmp=compare): + args_order = [x['dest'] for x in common.GetArgsDict().values()] + for key in sorted(args, key=args_order.index): val = args[key] key_str = str(key).rjust(max_key_len) val_str = str(val) @@ -193,9 +192,9 @@ class BisectObject(Bisector): if options.dir: os.environ['BISECT_DIR'] = options.dir self.options.dir = os.environ.get('BISECT_DIR', '/tmp/sysroot_bisect') - self.setup_cmd = ('%s %s %s %s' % - (self.sysroot_wrapper_setup, self.options.board, - self.options.remote, self.options.package)) + self.setup_cmd = ( + '%s %s %s %s' % (self.sysroot_wrapper_setup, self.options.board, + self.options.remote, self.options.package)) self.ArgOverride(self.default_kwargs, overrides) @@ -306,7 +305,7 @@ def Run(bisector): _HELP_EPILOG = """ -Run ./bisect.py {method} --help for individual method help/args +Run ./run_bisect.py {method} --help for individual method help/args ------------------ @@ -320,7 +319,7 @@ def Main(argv): override_parser = argparse.ArgumentParser( add_help=False, argument_default=argparse.SUPPRESS, - usage='bisect.py {mode} [options]') + usage='run_bisect.py {mode} [options]') common.BuildArgParser(override_parser, override=True) epilog = _HELP_EPILOG + override_parser.format_help() diff --git a/binary_search_tool/run_bisect_test.py b/binary_search_tool/run_bisect_tests.py index d4ff4f73..9172d678 100755 --- a/binary_search_tool/run_bisect_test.py +++ b/binary_search_tool/run_bisect_tests.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Run full bisection test.""" from __future__ import print_function @@ -81,7 +86,7 @@ def verify_compiler_and_wrapper(): # We don't need to do any special setup if running inside a ChromeOS # chroot. if os.path.exists('/usr/bin/x86_64-cros-linux-gnu-gcc'): - return + return True message = """ *** IMPORTANT --- READ THIS CAREFULLY!! *** diff --git a/binary_search_tool/sysroot_wrapper/testing_test.py b/binary_search_tool/sysroot_wrapper/testing_test.py index a0d6ca10..b5ceec1f 100755 --- a/binary_search_tool/sysroot_wrapper/testing_test.py +++ b/binary_search_tool/sysroot_wrapper/testing_test.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Test for sysroot_wrapper bisector. All files in bad_files will be determined to be bad. This test was made for @@ -15,9 +20,9 @@ import os base_path = ('/var/cache/chromeos-chrome/chrome-src-internal/src/out_daisy/' 'Release/obj/') bad_files = [ - os.path.join(base_path, 'base/base.cpu.o'), os.path.join( - base_path, 'base/base.version.o'), os.path.join(base_path, - 'apps/apps.launcher.o') + os.path.join(base_path, 'base/base.cpu.o'), + os.path.join(base_path, 'base/base.version.o'), + os.path.join(base_path, 'apps/apps.launcher.o') ] bisect_dir = os.environ.get('BISECT_DIR', '/tmp/sysroot_bisect') diff --git a/binary_search_tool/test/__init__.py b/binary_search_tool/test/__init__.py index 8b137891..76500def 100644 --- a/binary_search_tool/test/__init__.py +++ b/binary_search_tool/test/__init__.py @@ -1 +1,4 @@ - +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/binary_search_tool/test/binary_search_tool_tester.py b/binary_search_tool/test/binary_search_tool_test.py index aff45a86..ca9313b5 100755 --- a/binary_search_tool/test/binary_search_tool_tester.py +++ b/binary_search_tool/test/binary_search_tool_test.py @@ -1,10 +1,12 @@ -#!/usr/bin/env python2 - -# Copyright 2018 The Chromium OS Authors. All rights reserved. +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Tests for bisecting tool.""" +from __future__ import division from __future__ import print_function __author__ = 'shenhan@google.com (Han Shen)' @@ -16,15 +18,15 @@ import unittest from cros_utils import command_executer from binary_search_tool import binary_search_state -from binary_search_tool import bisect +from binary_search_tool import run_bisect -import common -import gen_obj +from binary_search_tool.test import common +from binary_search_tool.test import gen_obj def GenObj(): obj_num = random.randint(100, 1000) - bad_obj_num = random.randint(obj_num / 100, obj_num / 20) + bad_obj_num = random.randint(obj_num // 100, obj_num // 20) if bad_obj_num == 0: bad_obj_num = 1 gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)]) @@ -38,10 +40,10 @@ def CleanObj(): class BisectTest(unittest.TestCase): - """Tests for bisect.py""" + """Tests for run_bisect.py""" def setUp(self): - with open('./is_setup', 'w'): + with open('./is_setup', 'w', encoding='utf-8'): pass try: @@ -57,8 +59,8 @@ class BisectTest(unittest.TestCase): except OSError: pass - class FullBisector(bisect.Bisector): - """Test bisector to test bisect.py with""" + class FullBisector(run_bisect.Bisector): + """Test bisector to test run_bisect.py with""" def __init__(self, options, overrides): super(BisectTest.FullBisector, self).__init__(options, overrides) @@ -81,14 +83,14 @@ class BisectTest(unittest.TestCase): return 0 def test_full_bisector(self): - ret = bisect.Run(self.FullBisector({}, {})) - self.assertEquals(ret, 0) + ret = run_bisect.Run(self.FullBisector({}, {})) + self.assertEqual(ret, 0) self.assertFalse(os.path.exists(common.OBJECTS_FILE)) self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) def check_output(self): _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( - ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' + ('grep "Bad items are: " logs/binary_search_tool_test.py.out | ' 'tail -n1')) ls = out.splitlines() self.assertEqual(len(ls), 1) @@ -113,7 +115,7 @@ class BisectingUtilsTest(unittest.TestCase): """Generate [100-1000] object files, and 1-5% of which are bad ones.""" GenObj() - with open('./is_setup', 'w'): + with open('./is_setup', 'w', encoding='utf-8'): pass try: @@ -146,7 +148,7 @@ class BisectingUtilsTest(unittest.TestCase): test_script='./is_good.py', prune=True, file_args=True) - self.assertEquals(ret, 0) + self.assertEqual(ret, 0) self.check_output() def test_arg_parse(self): @@ -156,7 +158,7 @@ class BisectingUtilsTest(unittest.TestCase): '--test_script', './is_good.py', '--prune', '--file_args' ] ret = binary_search_state.Main(args) - self.assertEquals(ret, 0) + self.assertEqual(ret, 0) self.check_output() def test_test_setup_script(self): @@ -178,7 +180,7 @@ class BisectingUtilsTest(unittest.TestCase): test_setup_script='./test_setup.py', prune=True, file_args=True) - self.assertEquals(ret, 0) + self.assertEqual(ret, 0) self.check_output() def test_bad_test_setup_script(self): @@ -196,15 +198,15 @@ class BisectingUtilsTest(unittest.TestCase): state_file = binary_search_state.STATE_FILE hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE) - with open(state_file, 'w') as f: + with open(state_file, 'w', encoding='utf-8') as f: f.write('test123') bss = binary_search_state.MockBinarySearchState() with self.assertRaises(binary_search_state.Error): bss.SaveState() - with open(state_file, 'r') as f: - self.assertEquals(f.read(), 'test123') + with open(state_file, 'r', encoding='utf-8') as f: + self.assertEqual(f.read(), 'test123') os.remove(state_file) @@ -243,9 +245,9 @@ class BisectingUtilsTest(unittest.TestCase): bss = None bss2 = binary_search_state.MockBinarySearchState.LoadState() - self.assertEquals(bss2.all_items, test_items) - self.assertEquals(bss2.currently_good_items, set([])) - self.assertEquals(bss2.currently_bad_items, set([])) + self.assertEqual(bss2.all_items, test_items) + self.assertEqual(bss2.currently_good_items, set([])) + self.assertEqual(bss2.currently_bad_items, set([])) def test_tmp_cleanup(self): bss = binary_search_state.MockBinarySearchState( @@ -255,14 +257,14 @@ class BisectingUtilsTest(unittest.TestCase): bss.SwitchToGood(['0', '1', '2', '3']) tmp_file = None - with open('tmp_file', 'r') as f: + with open('tmp_file', 'r', encoding='utf-8') as f: tmp_file = f.read() os.remove('tmp_file') self.assertFalse(os.path.exists(tmp_file)) ws = common.ReadWorkingSet() for i in range(3): - self.assertEquals(ws[i], 42) + self.assertEqual(ws[i], 42) def test_verify_fail(self): bss = binary_search_state.MockBinarySearchState( @@ -298,11 +300,11 @@ class BisectingUtilsTest(unittest.TestCase): prune=False, file_args=True) bss.DoSearchBadItems() - self.assertEquals(len(bss.found_items), 1) + self.assertEqual(len(bss.found_items), 1) bad_objs = common.ReadObjectsFile() found_obj = int(bss.found_items.pop()) - self.assertEquals(bad_objs[found_obj], 1) + self.assertEqual(bad_objs[found_obj], 1) def test_set_file(self): binary_search_state.Run( @@ -326,12 +328,12 @@ class BisectingUtilsTest(unittest.TestCase): noincremental=True, file_args=True, verify=False) - self.assertEquals(ret, 0) + self.assertEqual(ret, 0) self.check_output() def check_output(self): _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( - ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' + ('grep "Bad items are: " logs/binary_search_tool_test.py.out | ' 'tail -n1')) ls = out.splitlines() self.assertEqual(len(ls), 1) @@ -354,7 +356,7 @@ class BisectingUtilsPassTest(BisectingUtilsTest): def check_pass_output(self, pass_name, pass_num, trans_num): _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( - ('grep "Bad pass: " logs/binary_search_tool_tester.py.out | ' + ('grep "Bad pass: " logs/binary_search_tool_test.py.out | ' 'tail -n1')) ls = out.splitlines() self.assertEqual(len(ls), 1) @@ -365,7 +367,7 @@ class BisectingUtilsPassTest(BisectingUtilsTest): _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( ('grep "Bad transformation number: ' - '" logs/binary_search_tool_tester.py.out | ' + '" logs/binary_search_tool_test.py.out | ' 'tail -n1')) ls = out.splitlines() self.assertEqual(len(ls), 1) @@ -383,7 +385,7 @@ class BisectingUtilsPassTest(BisectingUtilsTest): pass_bisect='./generate_cmd.py', prune=True, file_args=True) - self.assertEquals(ret, 1) + self.assertEqual(ret, 1) def test_gen_cmd_script(self): bss = binary_search_state.MockBinarySearchState( @@ -488,7 +490,7 @@ class BisectStressTest(unittest.TestCase): prune=True, file_args=True, verify=False) - self.assertEquals(ret, 0) + self.assertEqual(ret, 0) self.check_output() def test_every_index_is_bad(self): @@ -506,12 +508,12 @@ class BisectStressTest(unittest.TestCase): test_script='./is_good.py', prune=True, file_args=True) - self.assertEquals(ret, 0) + self.assertEqual(ret, 0) self.check_output() def check_output(self): _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( - ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' + ('grep "Bad items are: " logs/binary_search_tool_test.py.out | ' 'tail -n1')) ls = out.splitlines() self.assertEqual(len(ls), 1) diff --git a/binary_search_tool/test/cmd_script.py b/binary_search_tool/test/cmd_script.py index eb91fe9b..bfd56052 100755 --- a/binary_search_tool/test/cmd_script.py +++ b/binary_search_tool/test/cmd_script.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Command script without compiler support for pass level bisection. This script generates a pseudo log which a workable compiler should print out. @@ -11,7 +16,7 @@ from __future__ import print_function import os import sys -import common +from binary_search_tool.test import common def Main(argv): diff --git a/binary_search_tool/test/cmd_script_no_support.py b/binary_search_tool/test/cmd_script_no_support.py index a817f300..badbedc8 100644 --- a/binary_search_tool/test/cmd_script_no_support.py +++ b/binary_search_tool/test/cmd_script_no_support.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Command script without compiler support for pass level bisection. This script generates a pseudo log when certain bisecting flags are not @@ -13,8 +18,9 @@ import sys def Main(): if not os.path.exists('./is_setup'): return 1 - print('No support for -opt-bisect-limit or -print-debug-counter.', - file=sys.stderr) + print( + 'No support for -opt-bisect-limit or -print-debug-counter.', + file=sys.stderr) return 0 diff --git a/binary_search_tool/test/common.py b/binary_search_tool/test/common.py index 5c3ff538..cf5300f5 100755 --- a/binary_search_tool/test/common.py +++ b/binary_search_tool/test/common.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Common utility functions.""" DEFAULT_OBJECT_NUMBER = 1238 @@ -9,33 +14,29 @@ WORKING_SET_FILE = 'working_set.txt' def ReadWorkingSet(): working_set = [] - f = open(WORKING_SET_FILE, 'r') - for l in f: - working_set.append(int(l)) - f.close() + with open(WORKING_SET_FILE, 'r', encoding='utf-8') as f: + for l in f: + working_set.append(int(l)) return working_set def WriteWorkingSet(working_set): - f = open(WORKING_SET_FILE, 'w') - for o in working_set: - f.write('{0}\n'.format(o)) - f.close() + with open(WORKING_SET_FILE, 'w', encoding='utf-8') as f: + for o in working_set: + f.write('{0}\n'.format(o)) def ReadObjectsFile(): objects_file = [] - f = open(OBJECTS_FILE, 'r') - for l in f: - objects_file.append(int(l)) - f.close() + with open(OBJECTS_FILE, 'r', encoding='utf-8') as f: + for l in f: + objects_file.append(int(l)) return objects_file def ReadObjectIndex(filename): object_index = [] - f = open(filename, 'r') - for o in f: - object_index.append(int(o)) - f.close() + with open(filename, 'r', encoding='utf-8') as f: + for o in f: + object_index.append(int(o)) return object_index diff --git a/binary_search_tool/test/gen_init_list.py b/binary_search_tool/test/gen_init_list.py index 002fc352..bc5dd8fe 100755 --- a/binary_search_tool/test/gen_init_list.py +++ b/binary_search_tool/test/gen_init_list.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Prints out index for every object file, starting from 0.""" from __future__ import print_function @@ -6,7 +11,7 @@ from __future__ import print_function import sys from cros_utils import command_executer -import common +from binary_search_tool.test import common def Main(): diff --git a/binary_search_tool/test/gen_obj.py b/binary_search_tool/test/gen_obj.py index a2bc7d93..4f65c71b 100755 --- a/binary_search_tool/test/gen_obj.py +++ b/binary_search_tool/test/gen_obj.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Script to generate a list of object files. 0 represents a good object file. @@ -12,7 +17,7 @@ import os import random import sys -import common +from binary_search_tool.test import common def Main(argv): @@ -77,12 +82,11 @@ def Main(argv): if os.path.isfile(common.WORKING_SET_FILE): os.remove(common.WORKING_SET_FILE) - f = open(common.OBJECTS_FILE, 'w') - w = open(common.WORKING_SET_FILE, 'w') - for i in obj_list: - f.write('{0}\n'.format(i)) - w.write('{0}\n'.format(i)) - f.close() + with open(common.OBJECTS_FILE, 'w', encoding='utf-8') as f: + with open(common.WORKING_SET_FILE, 'w', encoding='utf-8') as w: + for i in obj_list: + f.write('{0}\n'.format(i)) + w.write('{0}\n'.format(i)) obj_num = len(obj_list) bad_obj_num = obj_list.count(1) diff --git a/binary_search_tool/test/generate_cmd.py b/binary_search_tool/test/generate_cmd.py index f6876eda..51b36b0a 100755 --- a/binary_search_tool/test/generate_cmd.py +++ b/binary_search_tool/test/generate_cmd.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Generate a virtual cmd script for pass level bisection. This is a required argument for pass level bisecting. For unit test, we use @@ -15,7 +20,7 @@ def Main(): if not os.path.exists('./is_setup'): return 1 file_name = 'cmd_script.sh' - with open(file_name, 'w') as f: + with open(file_name, 'w', encoding='utf-8') as f: f.write('Generated by generate_cmd.py') return 0 diff --git a/binary_search_tool/test/is_good.py b/binary_search_tool/test/is_good.py index a0be4a08..662921e8 100755 --- a/binary_search_tool/test/is_good.py +++ b/binary_search_tool/test/is_good.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Check to see if the working set produces a good executable.""" from __future__ import print_function @@ -6,7 +11,7 @@ from __future__ import print_function import os import sys -import common +from binary_search_tool.test import common def Main(): diff --git a/binary_search_tool/test/is_good_noinc_prune.py b/binary_search_tool/test/is_good_noinc_prune.py index a900bd32..c0e42bb1 100755 --- a/binary_search_tool/test/is_good_noinc_prune.py +++ b/binary_search_tool/test/is_good_noinc_prune.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Check to see if the working set produces a good executable. This test script is made for the noincremental-prune test. This makes sure @@ -12,16 +17,16 @@ from __future__ import print_function import os import sys -import common +from binary_search_tool.test import common def Main(): working_set = common.ReadWorkingSet() - with open('noinc_prune_good', 'r') as good_args: + with open('noinc_prune_good', 'r', encoding='utf-8') as good_args: num_good_args = len(good_args.readlines()) - with open('noinc_prune_bad', 'r') as bad_args: + with open('noinc_prune_bad', 'r', encoding='utf-8') as bad_args: num_bad_args = len(bad_args.readlines()) num_args = num_good_args + num_bad_args diff --git a/binary_search_tool/test/switch_tmp.py b/binary_search_tool/test/switch_tmp.py index 51b7110e..0f3c4234 100755 --- a/binary_search_tool/test/switch_tmp.py +++ b/binary_search_tool/test/switch_tmp.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Change portions of the object files to good. This file is a test switch script. Used only for the test test_tmp_cleanup. @@ -11,7 +16,7 @@ from __future__ import print_function import sys -import common +from binary_search_tool.test import common def Main(argv): @@ -23,7 +28,7 @@ def Main(argv): working_set[int(oi)] = 42 common.WriteWorkingSet(working_set) - with open('tmp_file', 'w') as f: + with open('tmp_file', 'w', encoding='utf-8') as f: f.write(argv[1]) return 0 diff --git a/binary_search_tool/test/switch_to_bad.py b/binary_search_tool/test/switch_to_bad.py index a1b6bd59..e3553eb6 100755 --- a/binary_search_tool/test/switch_to_bad.py +++ b/binary_search_tool/test/switch_to_bad.py @@ -1,11 +1,16 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Switch part of the objects file in working set to (possible) bad ones.""" from __future__ import print_function import sys -import common +from binary_search_tool.test import common def Main(argv): diff --git a/binary_search_tool/test/switch_to_bad_noinc_prune.py b/binary_search_tool/test/switch_to_bad_noinc_prune.py index db76acad..81b558e1 100755 --- a/binary_search_tool/test/switch_to_bad_noinc_prune.py +++ b/binary_search_tool/test/switch_to_bad_noinc_prune.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Switch part of the objects file in working set to (possible) bad ones. The "portion" is defined by the file (which is passed as the only argument to @@ -18,7 +23,7 @@ from __future__ import print_function import shutil import sys -import common +from binary_search_tool.test import common def Main(argv): diff --git a/binary_search_tool/test/switch_to_bad_set_file.py b/binary_search_tool/test/switch_to_bad_set_file.py index edf226d3..5b941c62 100755 --- a/binary_search_tool/test/switch_to_bad_set_file.py +++ b/binary_search_tool/test/switch_to_bad_set_file.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Switch part of the objects file in working set to (possible) bad ones. This script is meant to be specifically used with the set_file test. This uses @@ -10,7 +15,7 @@ from __future__ import print_function import os import sys -import common +from binary_search_tool.test import common def Main(_): diff --git a/binary_search_tool/test/switch_to_good.py b/binary_search_tool/test/switch_to_good.py index 59a118c1..97479329 100755 --- a/binary_search_tool/test/switch_to_good.py +++ b/binary_search_tool/test/switch_to_good.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Change portions of the object files to good. The "portion" is defined by the file (which is passed as the only argument to @@ -10,7 +15,7 @@ from __future__ import print_function import sys -import common +from binary_search_tool.test import common def Main(argv): diff --git a/binary_search_tool/test/switch_to_good_noinc_prune.py b/binary_search_tool/test/switch_to_good_noinc_prune.py index 00488a74..0b91a0d8 100755 --- a/binary_search_tool/test/switch_to_good_noinc_prune.py +++ b/binary_search_tool/test/switch_to_good_noinc_prune.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Change portions of the object files to good. The "portion" is defined by the file (which is passed as the only argument to @@ -18,7 +23,7 @@ from __future__ import print_function import shutil import sys -import common +from binary_search_tool.test import common def Main(argv): diff --git a/binary_search_tool/test/switch_to_good_set_file.py b/binary_search_tool/test/switch_to_good_set_file.py index b5e521f9..1cb05e0c 100755 --- a/binary_search_tool/test/switch_to_good_set_file.py +++ b/binary_search_tool/test/switch_to_good_set_file.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Change portions of the object files to good. The "portion" is defined by the file (which is passed as the only argument to @@ -14,7 +19,7 @@ from __future__ import print_function import os import sys -import common +from binary_search_tool.test import common def Main(_): diff --git a/binary_search_tool/test/test_setup.py b/binary_search_tool/test/test_setup.py index 0d6a410e..ecc8eb97 100755 --- a/binary_search_tool/test/test_setup.py +++ b/binary_search_tool/test/test_setup.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Emulate running of test setup script, is_good.py should fail without this.""" from __future__ import print_function @@ -8,7 +13,7 @@ import sys def Main(): # create ./is_setup - with open('./is_setup', 'w'): + with open('./is_setup', 'w', encoding='utf-8'): pass return 0 diff --git a/binary_search_tool/test/test_setup_bad.py b/binary_search_tool/test/test_setup_bad.py index d715f57a..cbca3c21 100755 --- a/binary_search_tool/test/test_setup_bad.py +++ b/binary_search_tool/test/test_setup_bad.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Emulate test setup that fails (i.e. failed flash to device)""" from __future__ import print_function diff --git a/build_chromeos.py b/build_chromeos.py index 0b0676d1..e275da1f 100755 --- a/build_chromeos.py +++ b/build_chromeos.py @@ -1,6 +1,10 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # -# Copyright 2010 Google Inc. All Rights Reserved. +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Script to checkout the ChromeOS source. This script sets up the ChromeOS source in the given directory, matching a @@ -92,8 +96,8 @@ def Main(argv): dest='debug', default=False, action='store_true', - help=("Optional. Build chrome browser with \"-g -O0\". " - "Notice, this also turns on \'--dev\'. " + help=('Optional. Build chrome browser with "-g -O0". ' + "Notice, this also turns on '--dev'. " 'Defaults to False.')) parser.add_argument( '--env', dest='env', default='', help='Env to pass to build_packages.') @@ -133,7 +137,7 @@ def Main(argv): 'This flags is used internally by this script. ' 'Contact the author for more detail.')) - if options.rebuild == True: + if options.rebuild: build_packages_env += ' EXTRA_BOARD_FLAGS=-e' # EXTRA_BOARD_FLAGS=-e should clean up the object files for the chrome # browser but it doesn't. So do it here. @@ -166,10 +170,9 @@ def Main(argv): if not os.path.isdir(options.chromeos_root + '/chroot/build/' + options.board) or options.clobber_board: # Run build_tc.py from binary package - ret = cmd_executer.ChrootRunCommand(options.chromeos_root, - misc.GetSetupBoardCommand( - options.board, - force=options.clobber_board)) + ret = cmd_executer.ChrootRunCommand( + options.chromeos_root, + misc.GetSetupBoardCommand(options.board, force=options.clobber_board)) logger.GetLogger().LogFatalIf(ret, 'setup_board failed') else: logger.GetLogger().LogOutput('Did not setup_board ' @@ -179,23 +182,23 @@ def Main(argv): # Perform 2-step build_packages to build a debug chrome browser. # Firstly, build everything that chromeos-chrome depends on normally. - if options.rebuild == True: + if options.rebuild: # Give warning about "--rebuild" and "--debug". Under this combination, # only dependencies of "chromeos-chrome" get rebuilt. logger.GetLogger().LogWarning( - "\"--rebuild\" does not correctly re-build every package when " - "\"--debug\" is enabled. ") + '--rebuild" does not correctly re-build every package when ' + '"--debug" is enabled. ') # Replace EXTRA_BOARD_FLAGS=-e with "-e --onlydeps" build_packages_env = build_packages_env.replace( - 'EXTRA_BOARD_FLAGS=-e', 'EXTRA_BOARD_FLAGS=\"-e --onlydeps\"') + 'EXTRA_BOARD_FLAGS=-e', 'EXTRA_BOARD_FLAGS="-e --onlydeps"') else: build_packages_env += ' EXTRA_BOARD_FLAGS=--onlydeps' ret = cmd_executer.ChrootRunCommand( - options.chromeos_root, "CFLAGS=\"$(portageq-%s envvar CFLAGS) %s\" " - "CXXFLAGS=\"$(portageq-%s envvar CXXFLAGS) %s\" " - "LDFLAGS=\"$(portageq-%s envvar LDFLAGS) %s\" " + options.chromeos_root, 'CFLAGS="$(portageq-%s envvar CFLAGS) %s" ' + 'CXXFLAGS="$(portageq-%s envvar CXXFLAGS) %s" ' + 'LDFLAGS="$(portageq-%s envvar LDFLAGS) %s" ' 'CHROME_ORIGIN=SERVER_SOURCE ' '%s ' '%s --skip_chroot_upgrade' @@ -208,16 +211,16 @@ def Main(argv): # Secondly, build chromeos-chrome using debug mode. # Replace '--onlydeps' with '--nodeps'. - if options.rebuild == True: + if options.rebuild: build_packages_env = build_packages_env.replace( - 'EXTRA_BOARD_FLAGS=\"-e --onlydeps\"', 'EXTRA_BOARD_FLAGS=--nodeps') + 'EXTRA_BOARD_FLAGS="-e --onlydeps"', 'EXTRA_BOARD_FLAGS=--nodeps') else: build_packages_env = build_packages_env.replace( 'EXTRA_BOARD_FLAGS=--onlydeps', 'EXTRA_BOARD_FLAGS=--nodeps') ret = cmd_executer.ChrootRunCommand( - options.chromeos_root, "CFLAGS=\"$(portageq-%s envvar CFLAGS) %s\" " - "CXXFLAGS=\"$(portageq-%s envvar CXXFLAGS) %s\" " - "LDFLAGS=\"$(portageq-%s envvar LDFLAGS) %s\" " + options.chromeos_root, 'CFLAGS="$(portageq-%s envvar CFLAGS) %s" ' + 'CXXFLAGS="$(portageq-%s envvar CXXFLAGS) %s" ' + 'LDFLAGS="$(portageq-%s envvar LDFLAGS) %s" ' 'CHROME_ORIGIN=SERVER_SOURCE BUILDTYPE=Debug ' '%s ' '%s --skip_chroot_upgrade' @@ -237,11 +240,11 @@ def Main(argv): # Up to now, we have a debug built chromos-chrome browser. # Fall through to build the rest of the world. - # Build packages + # Build packages ret = cmd_executer.ChrootRunCommand( - options.chromeos_root, "CFLAGS=\"$(portageq-%s envvar CFLAGS) %s\" " - "CXXFLAGS=\"$(portageq-%s envvar CXXFLAGS) %s\" " - "LDFLAGS=\"$(portageq-%s envvar LDFLAGS) %s\" " + options.chromeos_root, 'CFLAGS="$(portageq-%s envvar CFLAGS) %s" ' + 'CXXFLAGS="$(portageq-%s envvar CXXFLAGS) %s" ' + 'LDFLAGS="$(portageq-%s envvar LDFLAGS) %s" ' 'CHROME_ORIGIN=SERVER_SOURCE ' '%s ' '%s --skip_chroot_upgrade' % @@ -261,19 +264,18 @@ def Main(argv): flags_file_name = 'flags.txt' flags_file_path = ('%s/src/build/images/%s/latest/%s' % (options.chromeos_root, options.board, flags_file_name)) - flags_file = open(flags_file_path, 'wb') - flags_file.write('CFLAGS=%s\n' % options.cflags) - flags_file.write('CXXFLAGS=%s\n' % options.cxxflags) - flags_file.write('LDFLAGS=%s\n' % options.ldflags) - flags_file.close() + with open(flags_file_path, 'w', encoding='utf-8') as flags_file: + flags_file.write('CFLAGS=%s\n' % options.cflags) + flags_file.write('CXXFLAGS=%s\n' % options.cxxflags) + flags_file.write('LDFLAGS=%s\n' % options.ldflags) if options.label: image_dir_path = ('%s/src/build/images/%s/latest' % (options.chromeos_root, options.board)) real_image_dir_path = os.path.realpath(image_dir_path) - command = ('ln -sf -T %s %s/%s' % - (os.path.basename(real_image_dir_path), - os.path.dirname(real_image_dir_path), options.label)) + command = ('ln -sf -T %s %s/%s' % (os.path.basename(real_image_dir_path), + os.path.dirname(real_image_dir_path), + options.label)) ret = cmd_executer.RunCommand(command) logger.GetLogger().LogFatalIf( diff --git a/build_tc.py b/build_tc.py index c14b6905..9b90f55c 100755 --- a/build_tc.py +++ b/build_tc.py @@ -1,8 +1,9 @@ -#!/usr/bin/env python2 -# +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # Copyright 2010 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Script to build the ChromeOS toolchain. This script sets up the toolchain if you give it the gcctools directory. @@ -100,8 +101,9 @@ class ToolchainPart(object): command = 'mkdir -p %s' % build_dir self._ce.RunCommand(command) - mounted_build_dir = os.path.join(self._chromeos_root, 'chroot', '%s-%s' % - (self._chroot_source_path, build_suffix)) + mounted_build_dir = os.path.join( + self._chromeos_root, 'chroot', + '%s-%s' % (self._chroot_source_path, build_suffix)) build_mp = tc_enter_chroot.MountPoint(build_dir, mounted_build_dir, getpass.getuser()) mount_points.append(build_mp) @@ -148,10 +150,10 @@ class ToolchainPart(object): if self._name == 'gcc' and not self._gcc_enable_ccache: env['USE'] += ' -wrapper_ccache' - env['%s_SOURCE_PATH' % self._name.upper()] = (os.path.join( - '/', self._chroot_source_path)) + env['%s_SOURCE_PATH' % self._name.upper()] = ( + os.path.join('/', self._chroot_source_path)) env['ACCEPT_KEYWORDS'] = '~*' - env_string = ' '.join(["%s=\"%s\"" % var for var in env.items()]) + env_string = ' '.join(['%s="%s"' % var for var in env.items()]) command = 'emerge =cross-%s/%s-9999' % (self._ctarget, self._name) full_command = 'sudo %s %s' % (env_string, command) rv = self._ce.ChrootRunCommand(self._chromeos_root, full_command) diff --git a/buildbot_test_llvm.py b/buildbot_test_llvm.py index 7e7c6e63..968c67b8 100755 --- a/buildbot_test_llvm.py +++ b/buildbot_test_llvm.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2017 The Chromium OS Authors. All rights reserved. @@ -125,7 +125,7 @@ def WriteRotatingReportsData(results_dict, date): """Write data for waterfall report.""" fname = '%d-%02d-%02d.builds' % (date.year, date.month, date.day) filename = os.path.join(DATA_DIR, 'rotating-builders', fname) - with open(filename, 'w') as out_file: + with open(filename, 'w', encoding='utf-8') as out_file: for board in results_dict.keys(): buildbucket_id = results_dict[board] out_file.write('%s,%s\n' % (buildbucket_id, board)) @@ -172,7 +172,7 @@ def Main(argv): if options.board: fv = ToolchainVerifier(options.board, options.chromeos_root, options.weekday, options.patches, options.compiler) - return fv.Doall() + return fv.DoAll() today = datetime.date.today() delta = today - START_DATE @@ -190,7 +190,7 @@ def Main(argv): results_dict[board] = buildbucket_id except SystemExit: logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board) - with open(logfile, 'w') as f: + with open(logfile, 'w', encoding='utf-8') as f: f.write('Verifier got an exception, please check the log.\n') WriteRotatingReportsData(results_dict, today) diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py index 175dae5e..5f7c21d3 100755 --- a/buildbot_test_toolchains.py +++ b/buildbot_test_toolchains.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2016 The Chromium OS Authors. All rights reserved. @@ -34,11 +34,12 @@ from cros_utils import buildbot_utils USE_LLVM_NEXT_PATCH = '513590' CROSTC_ROOT = '/usr/local/google/crostc' +NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly-tests') ROLE_ACCOUNT = 'mobiletc-prebuild' TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) MAIL_PROGRAM = '~/var/bin/mail-sheriff' PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') -NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') +NIGHTLY_TESTS_RESULTS = os.path.join(CROSTC_ROOT, 'nightly_test_reports') IMAGE_DIR = '{board}-{image_type}' IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}' @@ -87,7 +88,7 @@ class ToolchainComparator(object): timestamp = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%H:%M:%S') self._reports_dir = os.path.join( - NIGHTLY_TESTS_DIR, + NIGHTLY_TESTS_RESULTS, '%s.%s' % (timestamp, board), ) @@ -145,7 +146,7 @@ class ToolchainComparator(object): Given the names of the trybot, vanilla and non-AFDO images, create the appropriate crosperf experiment file and launch crosperf on it. """ - experiment_file_dir = os.path.join(CROSTC_ROOT, self._weekday) + experiment_file_dir = os.path.join(NIGHTLY_TESTS_DIR, self._weekday) experiment_file_name = '%s_toolchain_experiment.txt' % self._board compiler_string = 'llvm' @@ -194,7 +195,7 @@ class ToolchainComparator(object): } """ - with open(experiment_file, 'w') as f: + with open(experiment_file, 'w', encoding='utf-8') as f: f.write(experiment_header) f.write(experiment_tests) @@ -250,11 +251,10 @@ class ToolchainComparator(object): ret = self._ce.RunCommand(command) if ret != 0: raise RuntimeError('Crosperf execution error!') - else: - # Copy json report to pending archives directory. - command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) - ret = self._ce.RunCommand(command) - return + + # Copy json report to pending archives directory. + command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) + ret = self._ce.RunCommand(command) def _SendEmail(self): """Find email message generated by crosperf and send it.""" diff --git a/chromiumos_image_diff.py b/chromiumos_image_diff.py index 74906d32..66a54ccc 100755 --- a/chromiumos_image_diff.py +++ b/chromiumos_image_diff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2019 The Chromium OS Authors. All rights reserved. @@ -86,7 +86,7 @@ class CrosImage(object): '{r}/var {r}/mnt/stateful_partition {r}; sudo umount {s} ; ' 'rmdir {r} ; rmdir {s}\n').format( r=self.rootfs, s=self.stateful) - f = open(self.unmount_script, 'w') + f = open(self.unmount_script, 'w', encoding='utf-8') f.write(command) f.close() self._ce.RunCommand( @@ -160,9 +160,9 @@ class ImageComparator(object): i1 = self.images[0] i2 = self.images[1] t1 = i1.rootfs + '/' - elfset1 = set([e.replace(t1, '') for e in i1.elf_files]) + elfset1 = {e.replace(t1, '') for e in i1.elf_files} t2 = i2.rootfs + '/' - elfset2 = set([e.replace(t2, '') for e in i2.elf_files]) + elfset2 = {e.replace(t2, '') for e in i2.elf_files} dif1 = elfset1.difference(elfset2) msg = None if dif1: @@ -210,15 +210,15 @@ class ImageComparator(object): if full_path1 == full_path2: self.logger.LogError( - 'Error: We\'re comparing the SAME file - {0}'.format(f1)) + "Error: We're comparing the SAME file - {0}".format(f1)) continue command = ( 'objdump -d "{f1}" > {tempf1} ; ' 'objdump -d "{f2}" > {tempf2} ; ' # Remove path string inside the dissemble - 'sed -i \'s!{rootfs1}!!g\' {tempf1} ; ' - 'sed -i \'s!{rootfs2}!!g\' {tempf2} ; ' + "sed -i 's!{rootfs1}!!g' {tempf1} ; " + "sed -i 's!{rootfs2}!!g' {tempf2} ; " 'diff {tempf1} {tempf2} 1>/dev/null 2>&1').format( f1=full_path1, f2=full_path2, diff --git a/compiler_wrapper/build.py b/compiler_wrapper/build.py index 763b3e64..037b940f 100755 --- a/compiler_wrapper/build.py +++ b/compiler_wrapper/build.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -39,10 +39,16 @@ def calc_go_args(args, version): '-X', 'main.Version=' + version, ] + + # If the wrapper is intended for Chrome OS, we need to use libc's exec. + extra_args = [] + if 'cros' in args.config: + extra_args = ['-tags', 'libc_exec'] + return [ 'go', 'build', '-o', os.path.abspath(args.output_file), '-ldflags', ' '.join(ldFlags) - ] + ] + extra_args def read_version(build_dir): @@ -52,7 +58,7 @@ def read_version(build_dir): return r.read() last_commit_msg = subprocess.check_output( - ['git', '-C', build_dir, 'log', '-1', '--pretty=%B']) + ['git', '-C', build_dir, 'log', '-1', '--pretty=%B'], encoding='utf-8') # Use last found change id to support reverts as well. change_ids = re.findall(r'Change-Id: (\w+)', last_commit_msg) if not change_ids: diff --git a/compiler_wrapper/bundle.py b/compiler_wrapper/bundle.py index c1fa53e0..173625f4 100755 --- a/compiler_wrapper/bundle.py +++ b/compiler_wrapper/bundle.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -32,7 +32,7 @@ def copy_files(input_dir, output_dir): def read_change_id(input_dir): last_commit_msg = subprocess.check_output( - ['git', '-C', input_dir, 'log', '-1', '--pretty=%B']) + ['git', '-C', input_dir, 'log', '-1', '--pretty=%B'], encoding='utf-8') # Use last found change id to support reverts as well. change_ids = re.findall(r'Change-Id: (\w+)', last_commit_msg) if not change_ids: @@ -41,14 +41,15 @@ def read_change_id(input_dir): def write_readme(input_dir, output_dir, change_id): - with open(os.path.join(input_dir, 'bundle.README'), 'r') as r, \ - open(os.path.join(output_dir, 'README'), 'w') as w: - content = r.read() - w.write(content.format(change_id=change_id)) + with open( + os.path.join(input_dir, 'bundle.README'), 'r', encoding='utf-8') as r: + with open(os.path.join(output_dir, 'README'), 'w', encoding='utf-8') as w: + content = r.read() + w.write(content.format(change_id=change_id)) def write_version(output_dir, change_id): - with open(os.path.join(output_dir, 'VERSION'), 'w') as w: + with open(os.path.join(output_dir, 'VERSION'), 'w', encoding='utf-8') as w: w.write(change_id) diff --git a/compiler_wrapper/env.go b/compiler_wrapper/env.go index 4d83b17a..56b3a913 100644 --- a/compiler_wrapper/env.go +++ b/compiler_wrapper/env.go @@ -71,7 +71,7 @@ func (env *processEnv) stderr() io.Writer { } func (env *processEnv) exec(cmd *command) error { - return libcExec(env, cmd) + return execCmd(env, cmd) } func (env *processEnv) run(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { diff --git a/compiler_wrapper/go_exec.go b/compiler_wrapper/go_exec.go new file mode 100644 index 00000000..2f2e5ad9 --- /dev/null +++ b/compiler_wrapper/go_exec.go @@ -0,0 +1,23 @@ +// Copyright 2020 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build !libc_exec + +package main + +import ( + "os/exec" + "syscall" +) + +// Implement exec for users that don't need to dynamically link with glibc +// See b/144783188 and libc_exec.go. + +func execCmd(env env, cmd *command) error { + execCmd := exec.Command(cmd.Path, cmd.Args...) + mergedEnv := mergeEnvValues(env.environ(), cmd.EnvUpdates) + + ret := syscall.Exec(execCmd.Path, execCmd.Args, mergedEnv) + return newErrorwithSourceLocf("exec error: %v", ret) +} diff --git a/compiler_wrapper/goldenutil_test.go b/compiler_wrapper/goldenutil_test.go index 23e003c5..2b391d73 100644 --- a/compiler_wrapper/goldenutil_test.go +++ b/compiler_wrapper/goldenutil_test.go @@ -14,7 +14,6 @@ import ( "os" "path/filepath" "regexp" - "runtime" "strings" ) @@ -128,6 +127,11 @@ func fillGoldenResults(ctx *testContext, files []goldenFile) []goldenFile { } cmdResult := record.Cmds[len(newCmds)] cmdResult.Cmd = cmd + if numEnvUpdates := len(cmdResult.Cmd.EnvUpdates); numEnvUpdates > 0 { + if strings.HasPrefix(cmdResult.Cmd.EnvUpdates[numEnvUpdates-1], "PYTHONPATH") { + cmdResult.Cmd.EnvUpdates[numEnvUpdates-1] = "PYTHONPATH=/somepath/test_binary" + } + } newCmds = append(newCmds, cmdResult) io.WriteString(stdout, cmdResult.Stdout) io.WriteString(stderr, cmdResult.Stderr) @@ -146,13 +150,7 @@ func fillGoldenResults(ctx *testContext, files []goldenFile) []goldenFile { // Create an empty wrapper at the given path. // Needed as we are resolving symlinks which stats the wrapper file. ctx.writeFile(record.WrapperCmd.Cmd.Path, "") - // Assign a fixed path to os.Args[0] to pass the test. - tmp := os.Args[0] - // callCompiler verifies os.Args[0] exists, so use a real file. - _, file, _, _ := runtime.Caller(1) - os.Args[0] = file record.WrapperCmd.ExitCode = callCompiler(ctx, ctx.cfg, record.WrapperCmd.Cmd) - os.Args[0] = tmp if hasInternalError(ctx.stderrString()) { ctx.t.Errorf("found an internal error for wrapperCmd %#v and env #%v. Got: %s", record.WrapperCmd.Cmd, record.Env, ctx.stderrString()) diff --git a/compiler_wrapper/libc_exec.go b/compiler_wrapper/libc_exec.go index 268221dc..f8db9d86 100644 --- a/compiler_wrapper/libc_exec.go +++ b/compiler_wrapper/libc_exec.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// +build libc_exec + package main // #include <errno.h> @@ -26,7 +28,10 @@ import ( // LD_PRELOAD to work properly (e.g. gentoo sandbox). // Note that this changes the go binary to be a dynamically linked one. // See crbug.com/1000863 for details. -func libcExec(env env, cmd *command) error { +// To use this version of exec, please add '-tags libc_exec' when building Go binary. +// Without the tags, libc_exec.go will be used. + +func execCmd(env env, cmd *command) error { freeList := []unsafe.Pointer{} defer func() { for _, ptr := range freeList { diff --git a/compiler_wrapper/testdata/android_golden/bisect.json b/compiler_wrapper/testdata/android_golden/bisect.json index be07da68..25df7f17 100644 --- a/compiler_wrapper/testdata/android_golden/bisect.json +++ b/compiler_wrapper/testdata/android_golden/bisect.json @@ -27,7 +27,7 @@ "main.cc" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -62,7 +62,7 @@ "main.cc" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -100,7 +100,7 @@ "main.cc" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json index ad4feac3..28adfee2 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json @@ -41,7 +41,7 @@ "-Wno-implicit-int-float-conversion" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -90,7 +90,7 @@ "-Wno-implicit-int-float-conversion" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -142,7 +142,7 @@ "-Wno-implicit-int-float-conversion" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index 6e1d4981..b00f9740 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -55,7 +55,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -118,7 +118,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -184,7 +184,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index 6e1d4981..b00f9740 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -55,7 +55,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -118,7 +118,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -184,7 +184,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index 3fafcf01..5075f843 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -50,7 +50,7 @@ "x86_64-cros-linux-gnu" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -108,7 +108,7 @@ "x86_64-cros-linux-gnu" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -169,7 +169,7 @@ "x86_64-cros-linux-gnu" ], "env_updates": [ - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index 7d2acdac..cb5dea01 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -47,7 +47,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -102,7 +102,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] } } @@ -160,7 +160,7 @@ "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", "CCACHE_CPP2=yes", - "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" + "PYTHONPATH=/somepath/test_binary" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/update_compiler_wrapper.sh b/compiler_wrapper/update_compiler_wrapper.sh new file mode 100755 index 00000000..93de2be4 --- /dev/null +++ b/compiler_wrapper/update_compiler_wrapper.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# This script rebuilds and installs compiler wrappers + +if [[ ! -e /etc/cros_chroot_version ]]; then + echo "Please run this script inside chroot" + exit 1 +fi +set -e +cd "$(dirname "$(readlink -m "$0")")" +echo "Updated files:" +# Update the host wrapper +./build.py --config=cros.host --use_ccache=false --use_llvm_next=false --output_file=./clang_host_wrapper +sudo mv ./clang_host_wrapper /usr/bin/clang_host_wrapper +echo "/usr/bin/clang_host_wrapper" +sudo cp ../binary_search_tool/bisect_driver.py /usr/bin +echo "/usr/bin/clang_host_wrapper/bisect_driver.py" +# Update the target wrappers +for GCC in cross-x86_64-cros-linux-gnu/gcc cross-armv7a-cros-linux-gnueabihf/gcc cross-aarch64-cros-linux-gnu/gcc; do + FILES="$(equery f $GCC)" + ./build.py --config=cros.hardened --use_ccache=false --use_llvm_next=false --output_file=./sysroot_wrapper.hardened.noccache + sudo mv ./sysroot_wrapper.hardened.noccache "$(grep sysroot_wrapper.hardened.noccache <<< "${FILES}")" + echo "$(grep sysroot_wrapper.hardened.noccache <<< "${FILES}")" + ./build.py --config=cros.hardened --use_ccache=true --use_llvm_next=false --output_file=./sysroot_wrapper.hardened.ccache + sudo mv ./sysroot_wrapper.hardened.ccache "$(grep sysroot_wrapper.hardened.ccache <<< "${FILES}")" + echo "$(grep sysroot_wrapper.hardened.ccache <<< "${FILES}")" + sudo cp ../binary_search_tool/bisect_driver.py "$(grep bisect_driver.py <<< "${FILES}")" + echo "$(grep bisect_driver.py <<< "${FILES}")" +done diff --git a/command_executer_timeout_test.py b/cros_utils/command_executer_timeout_test.py index 26f39334..1c9c74cd 100755 --- a/command_executer_timeout_test.py +++ b/cros_utils/command_executer_timeout_test.py @@ -1,6 +1,10 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # -# Copyright 2010 Google Inc. All Rights Reserved. +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Timeout test for command_executer.""" from __future__ import print_function diff --git a/cros_utils/command_executer_unittest.py b/cros_utils/command_executer_unittest.py index 959000fd..22331ae0 100755 --- a/cros_utils/command_executer_unittest.py +++ b/cros_utils/command_executer_unittest.py @@ -11,7 +11,7 @@ from __future__ import print_function import time import unittest -import command_executer +from cros_utils import command_executer class CommandExecuterTest(unittest.TestCase): diff --git a/cros_utils/email_sender.py b/cros_utils/email_sender.py index 0019982e..6b8893ea 100755 --- a/cros_utils/email_sender.py +++ b/cros_utils/email_sender.py @@ -9,17 +9,35 @@ from __future__ import print_function -from email import encoders as Encoders -from email.mime.base import MIMEBase -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText +import base64 +import contextlib +import datetime import getpass +import json import os import smtplib import tempfile +from email import encoders as Encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from cros_utils import command_executer +X20_PATH = '/google/data/rw/teams/c-compiler-chrome/prod_emails' + + +@contextlib.contextmanager +def AtomicallyWriteFile(file_path): + temp_path = file_path + '.in_progress' + try: + with open(temp_path, 'w') as f: + yield f + os.rename(temp_path, file_path) + except: + os.remove(temp_path) + raise + class EmailSender(object): """Utility class to send email through SMTP or SendGMR.""" @@ -31,6 +49,93 @@ class EmailSender(object): self.name = name self.content = content + def SendX20Email(self, + subject, + identifier, + well_known_recipients=(), + direct_recipients=(), + text_body=None, + html_body=None): + """Enqueues an email in our x20 outbox. + + These emails ultimately get sent by the machinery in + //depot/google3/googleclient/chrome/chromeos_toolchain/mailer/mail.go. This + kind of sending is intended for accounts that don't have smtp or gmr access + (e.g., role accounts), but can be used by anyone with x20 access. + + All emails are sent from `mdb.c-compiler-chrome+${identifier}@google.com`. + + Args: + subject: email subject. Must be nonempty. + identifier: email identifier, or the text that lands after the `+` in the + "From" email address. Must be nonempty. + well_known_recipients: a list of well-known recipients for the email. + These are translated into addresses by our mailer. + Current potential values for this are ('sheriff', + 'cwp-team', 'cros-team', 'mage'). Either this or + direct_recipients must be a nonempty list. + direct_recipients: @google.com emails to send addresses to. Either this + or well_known_recipients must be a nonempty list. + text_body: a 'text/plain' email body to send. Either this or html_body + must be a nonempty string. Both may be specified + html_body: a 'text/html' email body to send. Either this or text_body + must be a nonempty string. Both may be specified + """ + # `str`s act a lot like tuples/lists. Ensure that we're not accidentally + # iterating over one of those (or anything else that's sketchy, for that + # matter). + if not isinstance(well_known_recipients, (tuple, list)): + raise ValueError('`well_known_recipients` is unexpectedly a %s' % + type(well_known_recipients)) + + if not isinstance(direct_recipients, (tuple, list)): + raise ValueError( + '`direct_recipients` is unexpectedly a %s' % type(direct_recipients)) + + if not subject or not identifier: + raise ValueError('both `subject` and `identifier` must be nonempty') + + if not (well_known_recipients or direct_recipients): + raise ValueError('either `well_known_recipients` or `direct_recipients` ' + 'must be specified') + + for recipient in direct_recipients: + if not recipient.endswith('@google.com'): + raise ValueError('All recipients must end with @google.com') + + if not (text_body or html_body): + raise ValueError('either `text_body` or `html_body` must be specified') + + email_json = { + 'email_identifier': identifier, + 'subject': subject, + } + + if well_known_recipients: + email_json['well_known_recipients'] = well_known_recipients + + if direct_recipients: + email_json['direct_recipients'] = direct_recipients + + if text_body: + email_json['body'] = text_body + + if html_body: + email_json['html_body'] = html_body + + # The name of this has two parts: + # - An easily sortable time, to provide uniqueness and let our emailer + # send things in the order they were put into the outbox. + # - 64 bits of entropy, so two racing email sends don't clobber the same + # file. + now = datetime.datetime.utcnow().isoformat('T', 'seconds') + 'Z' + entropy = base64.urlsafe_b64encode(os.getrandom(8)) + entropy_str = entropy.rstrip(b'=').decode('utf-8') + result_path = os.path.join(X20_PATH, now + '_' + entropy_str + '.json') + + with AtomicallyWriteFile(result_path) as f: + json.dump(email_json, f) + def SendEmail(self, email_to, subject, diff --git a/cros_utils/email_sender_unittest.py b/cros_utils/email_sender_unittest.py new file mode 100755 index 00000000..73492196 --- /dev/null +++ b/cros_utils/email_sender_unittest.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for email_sender.""" + +from __future__ import print_function + +import contextlib +import io +import json +import unittest +import unittest.mock as mock + +import cros_utils.email_sender as email_sender + + +class Test(unittest.TestCase): + """Tests for email_sender.""" + + @mock.patch('cros_utils.email_sender.AtomicallyWriteFile') + def test_x20_email_sending_rejects_invalid_inputs(self, write_file): + test_cases = [ + { + # no subject + 'subject': '', + 'identifier': 'foo', + 'direct_recipients': ['gbiv@google.com'], + 'text_body': 'hi', + }, + { + 'subject': 'foo', + # no identifier + 'identifier': '', + 'direct_recipients': ['gbiv@google.com'], + 'text_body': 'hi', + }, + { + 'subject': 'foo', + 'identifier': 'foo', + # no recipients + 'direct_recipients': [], + 'text_body': 'hi', + }, + { + 'subject': 'foo', + 'identifier': 'foo', + 'direct_recipients': ['gbiv@google.com'], + # no body + }, + { + 'subject': 'foo', + 'identifier': 'foo', + # direct recipients lack @google. + 'direct_recipients': ['gbiv'], + 'text_body': 'hi', + }, + { + 'subject': 'foo', + 'identifier': 'foo', + # non-list recipients + 'direct_recipients': 'gbiv@google.com', + 'text_body': 'hi', + }, + { + 'subject': 'foo', + 'identifier': 'foo', + # non-list recipients + 'well_known_recipients': 'sheriff', + 'text_body': 'hi', + }, + ] + + sender = email_sender.EmailSender() + for case in test_cases: + with self.assertRaises(ValueError): + sender.SendX20Email(**case) + + write_file.assert_not_called() + + @mock.patch('cros_utils.email_sender.AtomicallyWriteFile') + def test_x20_email_sending_translates_to_reasonable_json(self, write_file): + written_obj = None + + @contextlib.contextmanager + def actual_write_file(file_path): + nonlocal written_obj + + self.assertTrue( + file_path.startswith(email_sender.X20_PATH + '/'), file_path) + f = io.StringIO() + yield f + written_obj = json.loads(f.getvalue()) + + write_file.side_effect = actual_write_file + email_sender.EmailSender().SendX20Email( + subject='hello', + identifier='world', + well_known_recipients=['sheriff'], + direct_recipients=['gbiv@google.com'], + text_body='text', + html_body='html', + ) + + self.assertEqual( + written_obj, { + 'subject': 'hello', + 'email_identifier': 'world', + 'well_known_recipients': ['sheriff'], + 'direct_recipients': ['gbiv@google.com'], + 'body': 'text', + 'html_body': 'html', + }) + + +if __name__ == '__main__': + unittest.main() diff --git a/cros_utils/manifest_versions.py b/cros_utils/manifest_versions.py index 83e88908..115c6046 100644 --- a/cros_utils/manifest_versions.py +++ b/cros_utils/manifest_versions.py @@ -16,8 +16,8 @@ import shutil import tempfile import time -import command_executer -import logger +from cros_utils import command_executer +from cros_utils import logger def IsCrosVersion(version): diff --git a/cros_utils/misc_test.py b/cros_utils/misc_test.py index cb7b8c89..21a545e9 100755 --- a/cros_utils/misc_test.py +++ b/cros_utils/misc_test.py @@ -14,7 +14,7 @@ __author__ = 'asharif@google.com (Ahmad Sharif)' import unittest # Local modules -import misc +from cros_utils import misc class UtilsTest(unittest.TestCase): diff --git a/cros_utils/perf_diff.py b/cros_utils/perf_diff.py index fed6a81d..b8ddb0c4 100755 --- a/cros_utils/perf_diff.py +++ b/cros_utils/perf_diff.py @@ -18,8 +18,8 @@ import functools import re import sys -import misc -import tabulator +from cros_utils import misc +from cros_utils import tabulator ROWS_TO_SHOW = 'Rows_to_show_in_the_perf_table' TOTAL_EVENTS = 'Total_events_of_this_profile' diff --git a/cros_utils/tabulator_test.py b/cros_utils/tabulator_test.py index 4c0f8677..227e2d70 100755 --- a/cros_utils/tabulator_test.py +++ b/cros_utils/tabulator_test.py @@ -14,7 +14,7 @@ __author__ = 'asharif@google.com (Ahmad Sharif)' import unittest # Local modules -import tabulator +from cros_utils import tabulator class TabulatorTest(unittest.TestCase): diff --git a/cros_utils/timeline_test.py b/cros_utils/timeline_test.py index 314cf3c4..8a10e549 100755 --- a/cros_utils/timeline_test.py +++ b/cros_utils/timeline_test.py @@ -13,7 +13,7 @@ __author__ = 'yunlian@google.com (Yunlian Jiang)' import time import unittest -import timeline +from cros_utils import timeline class TimeLineTest(unittest.TestCase): diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py index 74d461a6..b5912c11 100644 --- a/crosperf/benchmark_run.py +++ b/crosperf/benchmark_run.py @@ -178,12 +178,10 @@ class BenchmarkRun(threading.Thread): return machine def GetExtraAutotestArgs(self): - if self.benchmark.perf_args and self.benchmark.suite == 'telemetry': - self._logger.LogError('Telemetry does not support profiler.') - self.benchmark.perf_args = '' - - if self.benchmark.perf_args and self.benchmark.suite == 'test_that': - self._logger.LogError('test_that does not support profiler.') + if (self.benchmark.perf_args and + self.benchmark.suite != 'telemetry_Crosperf'): + self._logger.LogError( + 'Non-telemetry benchmark does not support profiler.') self.benchmark.perf_args = '' if self.benchmark.perf_args: diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py index 4cadc35c..ab863004 100755 --- a/crosperf/benchmark_run_unittest.py +++ b/crosperf/benchmark_run_unittest.py @@ -365,16 +365,12 @@ class BenchmarkRunTest(unittest.TestCase): '--profiler=custom_perf --profiler_args=\'perf_options="record -a -e ' 'cycles"\'') - self.test_benchmark.suite = 'telemetry' - result = br.GetExtraAutotestArgs() - self.assertEqual(result, '') - self.assertEqual(self.err_msg, 'Telemetry does not support profiler.') - self.test_benchmark.perf_args = 'record -e cycles' self.test_benchmark.suite = 'test_that' result = br.GetExtraAutotestArgs() self.assertEqual(result, '') - self.assertEqual(self.err_msg, 'test_that does not support profiler.') + self.assertEqual(self.err_msg, + 'Non-telemetry benchmark does not support profiler.') self.test_benchmark.perf_args = 'junk args' self.test_benchmark.suite = 'telemetry_Crosperf' diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 7d1689cc..4527db5f 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -296,7 +296,7 @@ class ExperimentFactory(object): iterations, rm_chroot_tmp, perf_args, - '', + 'crosperf_Wrapper', # Use client wrapper in Autotest show_all_results, retries, run_local=False, diff --git a/crosperf/mock_instance.py b/crosperf/mock_instance.py index 4271d8fd..842d6343 100644 --- a/crosperf/mock_instance.py +++ b/crosperf/mock_instance.py @@ -46,10 +46,10 @@ label2 = MockLabel( chrome_src=None) benchmark1 = Benchmark('benchmark1', 'autotest_name_1', 'autotest_args', 2, '', - perf_args, '', '') + perf_args, 'telemetry_Crosperf', '') benchmark2 = Benchmark('benchmark2', 'autotest_name_2', 'autotest_args', 2, '', - perf_args, '', '') + perf_args, 'telemetry_Crosperf', '') keyval = {} keyval[0] = { diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 1993a6c3..20ab2ad2 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -40,7 +40,9 @@ class BenchmarkSettings(Settings): 'times to get a stable result.')) self.AddField( TextField( - 'suite', default='', description='The type of the benchmark.')) + 'suite', + default='test_that', + description='The type of the benchmark.')) self.AddField( IntegerField( 'retries', diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index 12a87ae2..a6339034 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -24,7 +24,7 @@ class BenchmarkSettingsTest(unittest.TestCase): self.assertEqual(res.GetField('test_name'), '') self.assertEqual(res.GetField('test_args'), '') self.assertEqual(res.GetField('iterations'), 0) - self.assertEqual(res.GetField('suite'), '') + self.assertEqual(res.GetField('suite'), 'test_that') class LabelSettingsTest(unittest.TestCase): diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index 71ca7e7f..79ace20d 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -15,7 +15,6 @@ import shlex import time from cros_utils import command_executer -from cros_utils.device_setup_utils import DutWrapper TEST_THAT_PATH = '/usr/bin/test_that' # TODO: Need to check whether Skylab is installed and set up correctly. @@ -75,34 +74,12 @@ class SuiteRunner(object): def Run(self, cros_machine, label, benchmark, test_args, profiler_args): machine_name = cros_machine.name - if benchmark.suite != 'telemetry_Crosperf': - # Initialize command executer on DUT for test_that runs. - run_on_dut = DutWrapper( - label.chromeos_root, - machine_name, - logger=self.logger, - log_level=self.log_level, - ce=self._ce, - dut_config=self.dut_config) for i in range(0, benchmark.retries + 1): - # TODO: For telemetry_Crosperf run, device setup has been moved into - # server test script; for client runs, need to figure out wrapper to do - # it before running, now it is still setup here. - if benchmark.suite != 'telemetry_Crosperf': - wait_time = run_on_dut.SetupDevice() - # This is for accumulating wait time for test_that runs only, - # for telemetry_Cropserf runs, please refer to result_cache. - cros_machine.AddCooldownWaitTime(wait_time) - if label.skylab: ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args) - elif benchmark.suite == 'telemetry_Crosperf': - ret_tup = self.Telemetry_Crosperf_Run(machine_name, label, benchmark, - test_args, profiler_args) else: ret_tup = self.Test_That_Run(machine_name, label, benchmark, test_args, profiler_args) - if ret_tup[0] != 0: self.logger.LogOutput('benchmark %s failed. Retries left: %s' % (benchmark.name, benchmark.retries - i)) @@ -116,43 +93,94 @@ class SuiteRunner(object): break return ret_tup + def RemoveTelemetryTempFile(self, machine, chromeos_root): + filename = 'telemetry@%s' % machine + fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename) + if os.path.exists(fullname): + os.remove(fullname) + + def GenTestArgs(self, benchmark, test_args, profiler_args): + args_list = [] + + if benchmark.suite != 'telemetry_Crosperf' and profiler_args: + self.logger.LogFatal('Tests other than telemetry_Crosperf do not ' + 'support profiler.') + + if test_args: + # Strip double quotes off args (so we can wrap them in single + # quotes, to pass through to Telemetry). + if test_args[0] == '"' and test_args[-1] == '"': + test_args = test_args[1:-1] + args_list.append("test_args='%s'" % test_args) + + args_list.append(GetDutConfigArgs(self.dut_config)) + + if not (benchmark.suite == 'telemetry_Crosperf' or + benchmark.suite == 'crosperf_Wrapper'): + self.logger.LogWarning('Please make sure the server test has stage for ' + 'device setup.\n') + else: + args_list.append('test=%s' % benchmark.test_name) + if benchmark.suite == 'telemetry_Crosperf': + args_list.append('run_local=%s' % benchmark.run_local) + args_list.append(GetProfilerArgs(profiler_args)) + + return args_list + def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): """Run the test_that test..""" - options = [] - if label.board: - options.append('--board=%s' % label.board) - if test_args: - options.append(test_args) - if profiler_args: - self.logger.LogFatal('test_that does not support profiler.') + + # Remove existing test_that results command = 'rm -rf /usr/local/autotest/results/*' self._ce.CrosRunCommand( command, machine=machine, chromeos_root=label.chromeos_root) - autotest_dir = AUTOTEST_DIR - if label.autotest_path != '': - autotest_dir = label.autotest_path + if benchmark.suite == 'telemetry_Crosperf': + if not os.path.isdir(label.chrome_src): + self.logger.LogFatal('Cannot find chrome src dir to ' + 'run telemetry: %s' % label.chrome_src) + # Check for and remove temporary file that may have been left by + # previous telemetry runs (and which might prevent this run from + # working). + self.RemoveTelemetryTempFile(machine, label.chromeos_root) + + # --autotest_dir specifies which autotest directory to use. + autotest_dir_arg = '--autotest_dir=%s' % ( + label.autotest_path if label.autotest_path else AUTOTEST_DIR) + + # --fast avoids unnecessary copies of syslogs. + fast_arg = '--fast' + board_arg = '--board=%s' % label.board + + args_list = self.GenTestArgs(benchmark, test_args, profiler_args) + args_arg = '--args=%s' % pipes.quote(' '.join(args_list)) + + command = ' '.join([ + TEST_THAT_PATH, autotest_dir_arg, fast_arg, board_arg, args_arg, + machine, benchmark.suite if + (benchmark.suite == 'telemetry_Crosperf' or + benchmark.suite == 'crosperf_Wrapper') else benchmark.test_name + ]) - autotest_dir_arg = '--autotest_dir %s' % autotest_dir - # For non-telemetry tests, specify an autotest directory only if the - # specified directory is different from default (crosbug.com/679001). - if autotest_dir == AUTOTEST_DIR: - autotest_dir_arg = '' + # Use --no-ns-pid so that cros_sdk does not create a different + # process namespace and we can kill process created easily by their + # process group. + chrome_root_options = ('--no-ns-pid ' + '--chrome_root={} --chrome_root_mount={} ' + 'FEATURES="-usersandbox" ' + 'CHROME_ROOT={}'.format(label.chrome_src, + CHROME_MOUNT_DIR, + CHROME_MOUNT_DIR)) - command = (('%s %s --fast ' - '%s %s %s') % (TEST_THAT_PATH, autotest_dir_arg, - ' '.join(options), machine, benchmark.test_name)) if self.log_level != 'verbose': self.logger.LogOutput('Running test.') self.logger.LogOutput('CMD: %s' % command) - # Use --no-ns-pid so that cros_sdk does not create a different - # process namespace and we can kill process created easily by - # their process group. + return self._ce.ChrootRunCommandWOutput( label.chromeos_root, command, command_terminator=self._ct, - cros_sdk_options='--no-ns-pid') + cros_sdk_options=chrome_root_options) def DownloadResult(self, label, task_id): gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL) @@ -200,26 +228,9 @@ class SuiteRunner(object): # TODO: now only put toolchain pool here, user need to be able to specify # which pool to use. Need to request feature to not use this option at all. options.append('-pool=toolchain') - test_args_list = [] - if benchmark.suite == 'telemetry_Crosperf': - if test_args: - # Strip double quotes off args (so we can wrap them in single - # quotes, to pass through to Telemetry). - if test_args[0] == '"' and test_args[-1] == '"': - test_args_list.append(test_args[1:-1]) - if profiler_args: - test_args_list.append(GetProfilerArgs(profiler_args)) - if self.dut_config: - test_args_list.append(GetDutConfigArgs(self.dut_config)) - test_args_list.append('run_local=%s' % benchmark.run_local) - test_args_list.append('test=%s' % benchmark.test_name) - else: - if profiler_args: - self.logger.LogFatal('Client tests do not support profiler.') - if test_args: - test_args_list.append(test_args) - if test_args_list: - options.append('-test-args=%s' % pipes.quote(' '.join(test_args_list))) + + args_list = self.GenTestArgs(benchmark, test_args, profiler_args) + options.append('-test-args=%s' % pipes.quote(' '.join(args_list))) dimensions = [] for dut in label.remote: @@ -227,7 +238,9 @@ class SuiteRunner(object): command = (('%s create-test %s %s %s') % \ (SKYLAB_PATH, ' '.join(dimensions), ' '.join(options), - 'telemetry_Crosperf' if benchmark.suite == 'telemetry_Crosperf' + benchmark.suite if + (benchmark.suite == 'telemetry_Crosperf' or + benchmark.suite == 'crosperf_Wrapper') else benchmark.test_name)) if self.log_level != 'verbose': @@ -279,67 +292,6 @@ class SuiteRunner(object): return (ret_tup[0], result_dir, ret_tup[2]) return ret_tup - def RemoveTelemetryTempFile(self, machine, chromeos_root): - filename = 'telemetry@%s' % machine - fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename) - if os.path.exists(fullname): - os.remove(fullname) - - def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args, - profiler_args): - if not os.path.isdir(label.chrome_src): - self.logger.LogFatal('Cannot find chrome src dir to' - ' run telemetry: %s' % label.chrome_src) - - # Check for and remove temporary file that may have been left by - # previous telemetry runs (and which might prevent this run from - # working). - self.RemoveTelemetryTempFile(machine, label.chromeos_root) - - # For telemetry runs, we can use the autotest copy from the source - # location. No need to have one under /build/<board>. - autotest_dir_arg = '--autotest_dir %s' % AUTOTEST_DIR - if label.autotest_path != '': - autotest_dir_arg = '--autotest_dir %s' % label.autotest_path - - profiler_args = GetProfilerArgs(profiler_args) - dut_config_args = GetDutConfigArgs(self.dut_config) - # --fast avoids unnecessary copies of syslogs. - fast_arg = '--fast' - args_string = '' - if test_args: - # Strip double quotes off args (so we can wrap them in single - # quotes, to pass through to Telemetry). - if test_args[0] == '"' and test_args[-1] == '"': - test_args = test_args[1:-1] - args_string = "test_args='%s'" % test_args - - args = '{} run_local={} test={} {} {}'.format( - args_string, benchmark.run_local, benchmark.test_name, dut_config_args, - profiler_args) - - cmd = ('{} {} {} --board={} --args={} {} telemetry_Crosperf'.format( - TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board, - pipes.quote(args), machine)) - - # Use --no-ns-pid so that cros_sdk does not create a different - # process namespace and we can kill process created easily by their - # process group. - chrome_root_options = ('--no-ns-pid ' - '--chrome_root={} --chrome_root_mount={} ' - 'FEATURES="-usersandbox" ' - 'CHROME_ROOT={}'.format(label.chrome_src, - CHROME_MOUNT_DIR, - CHROME_MOUNT_DIR)) - if self.log_level != 'verbose': - self.logger.LogOutput('Running test.') - self.logger.LogOutput('CMD: %s' % cmd) - return self._ce.ChrootRunCommandWOutput( - label.chromeos_root, - cmd, - command_terminator=self._ct, - cros_sdk_options=chrome_root_options) - def CommandTerminator(self): return self._ct diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 7f3a7bc4..73fcb45b 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -10,8 +10,6 @@ from __future__ import print_function import json -import os.path -import time import unittest import unittest.mock as mock @@ -23,7 +21,6 @@ from benchmark import Benchmark from cros_utils import command_executer from cros_utils import logger -from cros_utils.device_setup_utils import DutWrapper from machine_manager import MockCrosMachine @@ -49,23 +46,21 @@ class SuiteRunnerTest(unittest.TestCase): 'telemetry_Crosperf', # suite True) # show_all_results - test_that_bench = Benchmark( + crosperf_wrapper_bench = Benchmark( 'b2_test', # name - 'octane', # test_name + 'webgl', # test_name '', # test_args 3, # iterations False, # rm_chroot_tmp - 'record -e cycles') # perf_args + '', # perf_args + 'crosperf_Wrapper') # suite def __init__(self, *args, **kwargs): super(SuiteRunnerTest, self).__init__(*args, **kwargs) - self.call_test_that_run = False self.skylab_run_args = [] self.test_that_args = [] - self.telemetry_run_args = [] - self.telemetry_crosperf_args = [] self.call_skylab_run = False - self.call_telemetry_crosperf_run = False + self.call_test_that_run = False def setUp(self): self.runner = suite_runner.SuiteRunner( @@ -79,30 +74,30 @@ class SuiteRunnerTest(unittest.TestCase): res = suite_runner.GetProfilerArgs(input_str) self.assertEqual(res, output_str) + def test_get_dut_config_args(self): + dut_config = {'enable_aslr': False, 'top_interval': 1.0} + output_str = ('dut_config=' + "'" + '{"enable_aslr": ' + 'false, "top_interval": 1.0}' + "'" + '') + res = suite_runner.GetDutConfigArgs(dut_config) + self.assertEqual(res, output_str) + def test_run(self): def reset(): + self.test_that_args = [] + self.skylab_run_args = [] self.call_test_that_run = False self.call_skylab_run = False - self.call_telemetry_crosperf_run = False - self.skylab_run_args = [] - self.test_that_args = [] - self.telemetry_run_args = [] - self.telemetry_crosperf_args = [] def FakeSkylabRun(test_label, benchmark, test_args, profiler_args): self.skylab_run_args = [test_label, benchmark, test_args, profiler_args] self.call_skylab_run = True return 'Ran FakeSkylabRun' - def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args, - profiler_args): - self.telemetry_crosperf_args = [ - machine, test_label, benchmark, test_args, profiler_args - ] - self.call_telemetry_crosperf_run = True - return 'Ran FakeTelemetryCrosperfRun' - def FakeTestThatRun(machine, test_label, benchmark, test_args, profiler_args): self.test_that_args = [ @@ -111,16 +106,8 @@ class SuiteRunnerTest(unittest.TestCase): self.call_test_that_run = True return 'Ran FakeTestThatRun' - def FakeRunner(command, ignore_status=False): - # pylint fix for unused variable. - del command, ignore_status - return 0, '', '' - self.runner.Skylab_Run = FakeSkylabRun - self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun self.runner.Test_That_Run = FakeTestThatRun - DutWrapper.SetupDevice = mock.Mock(return_value=0) - DutWrapper.RunCommandOnDut = mock.Mock(return_value=FakeRunner) self.runner.dut_config['enable_aslr'] = False self.runner.dut_config['cooldown_time'] = 0 @@ -133,36 +120,63 @@ class SuiteRunnerTest(unittest.TestCase): test_args = '' profiler_args = '' - reset() + # Test skylab run for telemetry_Crosperf and crosperf_Wrapper benchmarks. self.mock_label.skylab = True - self.runner.Run(cros_machine, self.mock_label, self.test_that_bench, + reset() + self.runner.Run(cros_machine, self.mock_label, self.crosperf_wrapper_bench, test_args, profiler_args) self.assertTrue(self.call_skylab_run) self.assertFalse(self.call_test_that_run) - self.assertFalse(self.call_telemetry_crosperf_run) self.assertEqual(self.skylab_run_args, - [self.mock_label, self.test_that_bench, '', '']) - DutWrapper.SetupDevice.assert_called_once() - self.mock_label.skylab = False + [self.mock_label, self.crosperf_wrapper_bench, '', '']) + + reset() + self.runner.Run(cros_machine, self.mock_label, + self.telemetry_crosperf_bench, test_args, profiler_args) + self.assertTrue(self.call_skylab_run) + self.assertFalse(self.call_test_that_run) + self.assertEqual(self.skylab_run_args, + [self.mock_label, self.telemetry_crosperf_bench, '', '']) + # Test test_that run for telemetry_Crosperf and crosperf_Wrapper benchmarks. + self.mock_label.skylab = False reset() - self.runner.Run(cros_machine, self.mock_label, self.test_that_bench, + self.runner.Run(cros_machine, self.mock_label, self.crosperf_wrapper_bench, test_args, profiler_args) self.assertTrue(self.call_test_that_run) - self.assertFalse(self.call_telemetry_crosperf_run) + self.assertFalse(self.call_skylab_run) self.assertEqual( self.test_that_args, - ['fake_machine', self.mock_label, self.test_that_bench, '', '']) + ['fake_machine', self.mock_label, self.crosperf_wrapper_bench, '', '']) reset() self.runner.Run(cros_machine, self.mock_label, self.telemetry_crosperf_bench, test_args, profiler_args) - self.assertFalse(self.call_test_that_run) - self.assertTrue(self.call_telemetry_crosperf_run) - self.assertEqual(self.telemetry_crosperf_args, [ + self.assertTrue(self.call_test_that_run) + self.assertFalse(self.call_skylab_run) + self.assertEqual(self.test_that_args, [ 'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', '' ]) + def test_gen_test_args(self): + test_args = '--iterations=2' + perf_args = 'record -a -e cycles' + + # Test crosperf_Wrapper benchmarks arg list generation + args_list = ["test_args='--iterations=2'", "dut_config='{}'", 'test=webgl'] + res = self.runner.GenTestArgs(self.crosperf_wrapper_bench, test_args, '') + self.assertCountEqual(res, args_list) + + # Test telemetry_Crosperf benchmarks arg list generation + args_list = [ + "test_args='--iterations=2'", "dut_config='{}'", 'test=octane', + 'run_local=False' + ] + args_list.append(suite_runner.GetProfilerArgs(perf_args)) + res = self.runner.GenTestArgs(self.telemetry_crosperf_bench, test_args, + perf_args) + self.assertCountEqual(res, args_list) + @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommandWOutput') @@ -176,10 +190,12 @@ class SuiteRunnerTest(unittest.TestCase): self.real_logger.LogMsg = FakeLogMsg self.runner.logger = self.real_logger + # Test crosperf_Wrapper benchmarks cannot take perf_args raised_exception = False try: self.runner.Test_That_Run('lumpy1.cros', self.mock_label, - self.test_that_bench, '', 'record -a -e cycles') + self.crosperf_wrapper_bench, '', + 'record -a -e cycles') except SystemExit: raised_exception = True self.assertTrue(raised_exception) @@ -188,7 +204,8 @@ class SuiteRunnerTest(unittest.TestCase): self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label, - self.test_that_bench, '--iterations=2', '') + self.crosperf_wrapper_bench, + '--iterations=2', '') self.assertEqual(mock_cros_runcmd.call_count, 1) self.assertEqual(mock_chroot_runcmd.call_count, 1) self.assertEqual(res, 0) @@ -197,48 +214,9 @@ class SuiteRunnerTest(unittest.TestCase): args_list = mock_chroot_runcmd.call_args_list[0][0] args_dict = mock_chroot_runcmd.call_args_list[0][1] self.assertEqual(len(args_list), 2) - self.assertEqual(args_list[0], '/tmp/chromeos') - self.assertEqual(args_list[1], ('/usr/bin/test_that ' - '--fast --board=lumpy ' - '--iterations=2 lumpy1.cros octane')) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) self.real_logger.LogMsg = save_log_msg - @mock.patch.object(os.path, 'isdir') - @mock.patch.object(command_executer.CommandExecuter, - 'ChrootRunCommandWOutput') - def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir): - - mock_isdir.return_value = True - mock_chroot_runcmd.return_value = 0 - self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd - profiler_args = ("--profiler=custom_perf --profiler_args='perf_options" - '="record -a -e cycles,instructions"\'') - self.runner.dut_config['turbostat'] = True - self.runner.dut_config['top_interval'] = 3 - res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label, - self.telemetry_crosperf_bench, '', - profiler_args) - self.assertEqual(res, 0) - self.assertEqual(mock_chroot_runcmd.call_count, 1) - args_list = mock_chroot_runcmd.call_args_list[0][0] - args_dict = mock_chroot_runcmd.call_args_list[0][1] - self.assertEqual(args_list[0], '/tmp/chromeos') - self.assertEqual( - args_list[1], - ('/usr/bin/test_that --autotest_dir ' - '/mnt/host/source/src/third_party/autotest/files --fast ' - "--board=lumpy --args=' run_local=False test=octane " - 'dut_config=\'"\'"\'{"turbostat": true, "top_interval": 3}\'"\'"\' ' - 'profiler=custom_perf profiler_args=\'"\'"\'record -a -e ' - 'cycles,instructions\'"\'"\'\' lumpy1.cros telemetry_Crosperf')) - self.assertEqual(args_dict['cros_sdk_options'], - ('--no-ns-pid --chrome_root= ' - '--chrome_root_mount=/tmp/chrome_root ' - 'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root')) - self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - self.assertEqual(len(args_dict), 2) - @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') @mock.patch.object(json, 'loads') def test_skylab_run_client(self, mock_json_loads, mock_runcmd): @@ -265,82 +243,21 @@ class SuiteRunnerTest(unittest.TestCase): self.mock_label.skylab = True self.runner.DownloadResult = FakeDownloadResult - res = self.runner.Skylab_Run(self.mock_label, self.test_that_bench, '', '') + res = self.runner.Skylab_Run(self.mock_label, self.crosperf_wrapper_bench, + '', '') ret_tup = (0, '\nResults placed in tmp/swarming-12345\n', '') self.assertEqual(res, ret_tup) self.assertEqual(mock_runcmd.call_count, 2) args_list = mock_runcmd.call_args_list[0][0] args_dict = mock_runcmd.call_args_list[0][1] - self.assertEqual(args_list[0], - ('/usr/local/bin/skylab create-test ' - '-dim dut_name:lumpy1 -dim dut_name:lumpy.cros2 ' - '-board=lumpy -image=build ' - '-pool=toolchain octane')) + self.assertEqual(len(args_list), 1) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) args_list = mock_runcmd.call_args_list[1][0] self.assertEqual(args_list[0], ('skylab wait-task 12345')) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') - @mock.patch.object(json, 'loads') - def test_skylab_run_telemetry_Crosperf(self, mock_json_loads, mock_runcmd): - - def FakeDownloadResult(l, task_id): - if l and task_id: - self.assertEqual(task_id, '12345') - return None - - mock_runcmd.return_value = ( - 0, - 'Created Swarming task https://swarming/task/b12345', - '', - ) - self.mock_cmd_exec.RunCommandWOutput = mock_runcmd - - mock_json_loads.return_value = { - 'child-results': [{ - 'success': True, - 'task-run-url': 'https://swarming/task?id=12345' - }] - } - self.mock_json.loads = mock_json_loads - - self.mock_label.skylab = True - self.runner.DownloadResult = FakeDownloadResult - self.runner.Skylab_Run(self.mock_label, self.telemetry_crosperf_bench, '', - '') - args_list = mock_runcmd.call_args_list[0][0] - self.assertEqual( - args_list[0], - ('/usr/local/bin/skylab create-test ' - '-dim dut_name:lumpy1 -dim dut_name:lumpy.cros2 ' - '-board=lumpy -image=build ' - "-pool=toolchain -test-args='run_local=False test=octane' " - 'telemetry_Crosperf')) - - @mock.patch.object(time, 'sleep') - @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') - def test_download_result(self, mock_runcmd, mock_sleep): - mock_runcmd.return_value = 0 - mock_sleep.return_value = 0 - self.mock_cmd_exec.RunCommand = mock_runcmd - - self.runner.DownloadResult(self.mock_label, '12345') - - self.assertEqual(mock_runcmd.call_count, 2) - cmd = mock_runcmd.call_args_list[0][0][0] - self.assertEqual(cmd, - ('/tmp/chromeos/src/chromium/depot_tools/gsutil.py ls ' - 'gs://chromeos-autotest-results/swarming-12345/' - 'autoserv_test')) - cmd = mock_runcmd.call_args_list[1][0][0] - self.assertEqual(cmd, - ('/tmp/chromeos/src/chromium/depot_tools/gsutil.py -mq ' - 'cp -r gs://chromeos-autotest-results/swarming-12345 ' - '/tmp/chromeos/chroot/tmp')) - if __name__ == '__main__': unittest.main() diff --git a/debug_info_test/check_cus.py b/debug_info_test/check_cus.py index f68fe9cb..d3cd6365 100644 --- a/debug_info_test/check_cus.py +++ b/debug_info_test/check_cus.py @@ -1,7 +1,12 @@ +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""check compile units.""" + +from __future__ import print_function + import os import subprocess @@ -9,59 +14,62 @@ import check_ngcc cu_checks = [check_ngcc.not_by_gcc] + def check_compile_unit(dso_path, producer, comp_path): - """check all compiler flags used to build the compile unit. + """check all compiler flags used to build the compile unit. + + Args: + dso_path: path to the elf/dso. + producer: DW_AT_producer contains the compiler command line. + comp_path: DW_AT_comp_dir + DW_AT_name. - Args: - dso_path: path to the elf/dso - producer: DW_AT_producer contains the compiler command line. - comp_path: DW_AT_comp_dir + DW_AT_name + Returns: + A set of failed tests. + """ + failed = set() + for c in cu_checks: + if not c(dso_path, producer, comp_path): + failed.add(c.__module__) - Returns: - A set of failed tests. - """ - failed = set() - for c in cu_checks: - if not c(dso_path, producer, comp_path): - failed.add(c.__module__) + return failed - return failed def check_compile_units(dso_path): - """check all compile units in the given dso. - - Args: - dso_path: path to the dso - Return: - True if everything looks fine otherwise False. - """ - - failed = set() - producer = '' - comp_path = '' - - readelf = subprocess.Popen(['readelf', '--debug-dump=info', - '--dwarf-depth=1', dso_path], - stdout=subprocess.PIPE, - stderr=open(os.devnull, 'w')) - for l in readelf.stdout: - if 'DW_TAG_compile_unit' in l: - if producer: - failed = failed.union(check_compile_unit(dso_path, producer, - comp_path)) - producer = '' - comp_path = '' - elif 'DW_AT_producer' in l: - producer = l - elif 'DW_AT_name' in l: - comp_path = os.path.join(comp_path, l.split(':')[-1].strip()) - elif 'DW_AT_comp_dir' in l: - comp_path = os.path.join(l.split(':')[-1].strip(), comp_path) - if producer: + """check all compile units in the given dso. + + Args: + dso_path: path to the dso. + + Returns: + True if everything looks fine otherwise False. + """ + + failed = set() + producer = '' + comp_path = '' + + readelf = subprocess.Popen( + ['readelf', '--debug-dump=info', '--dwarf-depth=1', dso_path], + stdout=subprocess.PIPE, + stderr=open(os.devnull, 'w'), + encoding='utf-8') + for l in readelf.stdout: + if 'DW_TAG_compile_unit' in l: + if producer: failed = failed.union(check_compile_unit(dso_path, producer, comp_path)) + producer = '' + comp_path = '' + elif 'DW_AT_producer' in l: + producer = l + elif 'DW_AT_name' in l: + comp_path = os.path.join(comp_path, l.split(':')[-1].strip()) + elif 'DW_AT_comp_dir' in l: + comp_path = os.path.join(l.split(':')[-1].strip(), comp_path) + if producer: + failed = failed.union(check_compile_unit(dso_path, producer, comp_path)) - if failed: - print('%s failed check: %s' % (dso_path, ' '.join(failed))) - return False + if failed: + print('%s failed check: %s' % (dso_path, ' '.join(failed))) + return False - return True + return True diff --git a/debug_info_test/check_exist.py b/debug_info_test/check_exist.py index 5e7cce19..dbb89127 100644 --- a/debug_info_test/check_exist.py +++ b/debug_info_test/check_exist.py @@ -1,90 +1,102 @@ +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""check whether intended components exists in the given dso.""" + +from __future__ import print_function + import os import subprocess from whitelist import is_whitelisted + def check_debug_info(dso_path, readelf_content): - """check whether debug info section exists in the elf file. + """Check whether debug info section exists in the elf file. + + Args: + dso_path: path to the dso. + readelf_content: debug info dumped by command readelf. - Args: - readelf: debug info dumped by command readelf + Returns: + True if debug info section exists, otherwise False. + """ - Returns: - True if debug info section exists, otherwise False. - """ + # Return True if it is whitelisted + if is_whitelisted('exist_debug_info', dso_path): + return True - # Return True if it is whitelisted - if is_whitelisted('exist_debug_info', dso_path): - return True + for l in readelf_content: + if 'debug_info' in l: + return True + return False - for l in readelf_content: - if 'debug_info' in l: - return True - return False def check_producer(dso_path, readelf_content): - """check whether DW_AT_producer exists in each compile unit. - - Args: - readelf: debug info dumped by command readelf - - Returns: - True if DW_AT_producer exists in each compile unit, otherwise False. - Notice: If no compile unit in DSO, also return True. - """ - - # Return True if it is whitelisted - if is_whitelisted('exist_producer', dso_path): - return True - - # Indicate if there is a producer under each cu - cur_producer = False - - first_cu = True - producer_exist = True - - for l in readelf_content: - if 'DW_TAG_compile_unit' in l: - if not first_cu and not cur_producer: - producer_exist = False - break - first_cu = False - cur_producer = False - elif 'DW_AT_producer' in l: - cur_producer = True - - # Check whether last producer of compile unit exists in the elf, - # also return True if no cu in the DSO. - if not first_cu and not cur_producer: + """Check whether DW_AT_producer exists in each compile unit. + + Args: + dso_path: path to the dso. + readelf_content: debug info dumped by command readelf. + + Returns: + True if DW_AT_producer exists in each compile unit, otherwise False. + Notice: If no compile unit in DSO, also return True. + """ + + # Return True if it is whitelisted + if is_whitelisted('exist_producer', dso_path): + return True + + # Indicate if there is a producer under each cu + cur_producer = False + + first_cu = True + producer_exist = True + + for l in readelf_content: + if 'DW_TAG_compile_unit' in l: + if not first_cu and not cur_producer: producer_exist = False + break + first_cu = False + cur_producer = False + elif 'DW_AT_producer' in l: + cur_producer = True + + # Check whether last producer of compile unit exists in the elf, + # also return True if no cu in the DSO. + if not first_cu and not cur_producer: + producer_exist = False + + return producer_exist - return producer_exist def check_exist_all(dso_path): - """check whether intended components exists in the given dso. + """check whether intended components exists in the given dso. - Args: - dso_path: path to the dso - Return: - True if everything looks fine otherwise False. - """ + Args: + dso_path: path to the dso. - readelf = subprocess.Popen(['readelf', '--debug-dump=info', - '--dwarf-depth=1', dso_path], - stdout=subprocess.PIPE, - stderr=open(os.devnull, 'w')) - readelf_content = list(readelf.stdout) + Returns: + True if everything looks fine otherwise False. + """ - exist_checks = [check_debug_info, check_producer] + readelf = subprocess.Popen( + ['readelf', '--debug-dump=info', '--dwarf-depth=1', dso_path], + stdout=subprocess.PIPE, + stderr=open(os.devnull, 'w'), + encoding='utf-8') + readelf_content = list(readelf.stdout) - for e in exist_checks: - if not e(dso_path, readelf_content): - check_failed = e.__module__ + ': ' + e.__name__ - print('%s failed check: %s' % (dso_path, check_failed)) - return False + exist_checks = [check_debug_info, check_producer] - return True + for e in exist_checks: + if not e(dso_path, readelf_content): + check_failed = e.__module__ + ': ' + e.__name__ + print('%s failed check: %s' % (dso_path, check_failed)) + return False + + return True diff --git a/debug_info_test/check_icf.py b/debug_info_test/check_icf.py index 4ac67dbd..a46968e7 100644 --- a/debug_info_test/check_icf.py +++ b/debug_info_test/check_icf.py @@ -1,47 +1,53 @@ +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""check whether chrome was built with identical code folding.""" + +from __future__ import print_function + import os import re import subprocess + def check_identical_code_folding(dso_path): - """check whether chrome was built with identical code folding. - - Args: - dso_path: path to the dso - Return: - False if the dso is chrome and it was not built with icf, - True otherwise. - """ - - if not dso_path.endswith('/chrome.debug'): - return True - - # Run 'nm' on the chrome binary and read the output. - nm = subprocess.Popen(['nm', dso_path], - stdout=subprocess.PIPE, - stderr=open(os.devnull, 'w')) - nm_output, _ = nm.communicate() - - # Search for addresses of text symbols. - text_addresses = re.findall('^[0-9a-f]+[ ]+[tT] ', - nm_output, - re.MULTILINE) - - # Calculate number of text symbols in chrome binary. - num_text_addresses = len(text_addresses) - - # Calculate number of unique text symbols in chrome binary. - num_unique_text_addresses = len(set(text_addresses)) - - # Check that the number of duplicate symbols is at least 10,000. - # - https://crbug.com/813272#c18 - if num_text_addresses-num_unique_text_addresses >= 10000: - return True - - print('%s was not built with ICF' % dso_path) - print(' num_text_addresses = %d' % num_text_addresses) - print(' num_unique_text_addresses = %d' % num_unique_text_addresses) - return False + """check whether chrome was built with identical code folding. + + Args: + dso_path: path to the dso. + + Returns: + False if the dso is chrome and it was not built with icf, + True otherwise. + """ + + if not dso_path.endswith('/chrome.debug'): + return True + + # Run 'nm' on the chrome binary and read the output. + nm = subprocess.Popen(['nm', dso_path], + stdout=subprocess.PIPE, + stderr=open(os.devnull, 'w'), + encoding='utf-8') + nm_output, _ = nm.communicate() + + # Search for addresses of text symbols. + text_addresses = re.findall('^[0-9a-f]+[ ]+[tT] ', nm_output, re.MULTILINE) + + # Calculate number of text symbols in chrome binary. + num_text_addresses = len(text_addresses) + + # Calculate number of unique text symbols in chrome binary. + num_unique_text_addresses = len(set(text_addresses)) + + # Check that the number of duplicate symbols is at least 10,000. + # - https://crbug.com/813272#c18 + if num_text_addresses - num_unique_text_addresses >= 10000: + return True + + print('%s was not built with ICF' % dso_path) + print(' num_text_addresses = %d' % num_text_addresses) + print(' num_unique_text_addresses = %d' % num_unique_text_addresses) + return False diff --git a/debug_info_test/check_ngcc.py b/debug_info_test/check_ngcc.py index eecbb85e..501bb988 100644 --- a/debug_info_test/check_ngcc.py +++ b/debug_info_test/check_ngcc.py @@ -1,26 +1,30 @@ +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Check whether the compile unit is not built by gcc.""" + +from __future__ import print_function + from whitelist import is_whitelisted -def not_by_gcc(dso_path, producer, comp_path): - """Check whether the compile unit is not built by gcc. - Args: - dso_path: path to the elf/dso - producer: DW_AT_producer contains the compiler command line. - comp_path: DW_AT_comp_dir + DW_AT_name +def not_by_gcc(dso_path, producer, comp_path): + """Check whether the compile unit is not built by gcc. - Returns: - False if compiled by gcc otherwise True - """ - if is_whitelisted('ngcc_comp_path', comp_path): - return True + Args: + dso_path: path to the elf/dso. + producer: DW_AT_producer contains the compiler command line. + comp_path: DW_AT_comp_dir + DW_AT_name. - if is_whitelisted('ngcc_dso_path', dso_path): - return True + Returns: + False if compiled by gcc otherwise True. + """ + if is_whitelisted('ngcc_comp_path', comp_path): + return True - if 'GNU C' in producer: - return False + if is_whitelisted('ngcc_dso_path', dso_path): return True + + return 'GNU C' not in producer diff --git a/debug_info_test/debug_info_test.py b/debug_info_test/debug_info_test.py index 4839e69c..ae7e9f48 100755 --- a/debug_info_test/debug_info_test.py +++ b/debug_info_test/debug_info_test.py @@ -1,9 +1,13 @@ -#!/usr/bin/python2 - +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Test for debug info.""" + +from __future__ import print_function + import os import subprocess import sys @@ -12,46 +16,52 @@ import check_icf import check_cus import check_exist -elf_checks = [check_exist.check_exist_all, - check_cus.check_compile_units, - check_icf.check_identical_code_folding] +elf_checks = [ + check_exist.check_exist_all, check_cus.check_compile_units, + check_icf.check_identical_code_folding +] + def scanelf(root): - """find ELFs in root + """Find ELFs in root. + + Args: + root: root dir to start with the search. + + Returns: + Filenames of ELFs in root. + """ + p = subprocess.Popen(['scanelf', '-y', '-B', '-F', '%F', '-R', root], + stdout=subprocess.PIPE, + encoding='utf-8') + return [l.strip() for l in p.stdout] - Args: - root: root dir to start with the search. - Returns: - Filenames of ELFs in root. - """ - p = subprocess.Popen(['scanelf', '-y', '-B', '-F', '%F', '-R', root], - stdout=subprocess.PIPE) - return [l.strip() for l in p.stdout] def Main(argv): - if len(argv) < 2: - print('usage: %s [file|dir]') - return 1 - - files = [] - cand = argv[1] - if os.path.isfile(cand): - files = [cand] - elif os.path.isdir(cand): - files = scanelf(cand) - else: - print('usage: %s [file|dir]') - return 1 - - failed = False - for f in files: - for c in elf_checks: - if not c(f): - failed = True - - if failed: - return 1 - return 0 + if len(argv) < 2: + print('usage: %s [file|dir]') + return 1 + + files = [] + cand = argv[1] + if os.path.isfile(cand): + files = [cand] + elif os.path.isdir(cand): + files = scanelf(cand) + else: + print('usage: %s [file|dir]') + return 1 + + failed = False + for f in files: + for c in elf_checks: + if not c(f): + failed = True + + if failed: + return 1 + return 0 + if __name__ == '__main__': - sys.exit(Main(sys.argv)) + sys.exit(Main(sys.argv)) diff --git a/debug_info_test/whitelist.py b/debug_info_test/whitelist.py index 383fcc3d..b53387a8 100644 --- a/debug_info_test/whitelist.py +++ b/debug_info_test/whitelist.py @@ -1,11 +1,17 @@ +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Whitelist functions.""" + +from __future__ import print_function + import os import glob import re + # Matching a string of length m in an NFA of size n is O(mn^2), but the # performance also depends largely on the implementation. It appears to be fast # enough according to the tests. @@ -13,45 +19,50 @@ import re # The performance bottleneck of this script is readelf. Unless this becomes # slower than readelf, don't waste time here. def is_whitelisted(list_name, pattern): - """chech whether the given pattern is specified in the whitelist. + """Check whether the given pattern is specified in the whitelist. + + Args: + list_name: name of the whitelist. + pattern: the target string. + + Returns: + True if matched otherwise False. + """ + return pattern and whitelists[list_name].match(pattern) - Args: - list_name: name of the whitelist - pattern: the target string - Returns: - True if matched otherwise False - """ - return pattern and whitelists[list_name].match(pattern) def prepare_whitelist(patterns): - """Join and compile the re patterns. + """Join and compile the re patterns. + + Args: + patterns: regex patterns. + + Returns: + A compiled re object. + """ + return re.compile('|'.join(patterns)) - Args: - patterns: regex patterns. - Return: - A compiled re object - """ - return re.compile('|'.join(patterns)) def load_whitelists(dirname): - """Load whitelists under dirname. - - A whitelist ends with .whitelist. - - Args: - dirname: path to the dir. - Returns: - A dictionary of 'filename' -> whitelist matcher. - """ - wlist = {} - for fn in glob.glob(os.path.join(dirname, '*.whitelist')): - key = os.path.splitext(os.path.basename(fn))[0] - with open(fn, 'r') as f: - patterns = f.read().splitlines() - patterns = [l for l in patterns if l != ''] - patterns = [l for l in patterns if l[0] != '#'] - wlist[key] = prepare_whitelist(patterns) - return wlist + """Load whitelists under dirname. + + A whitelist ends with .whitelist. + + Args: + dirname: path to the dir. + + Returns: + A dictionary of 'filename' -> whitelist matcher. + """ + wlist = {} + for fn in glob.glob(os.path.join(dirname, '*.whitelist')): + key = os.path.splitext(os.path.basename(fn))[0] + with open(fn, 'r', encoding='utf-8') as f: + patterns = f.read().splitlines() + patterns = [l for l in patterns if l != ''] + patterns = [l for l in patterns if l[0] != '#'] + wlist[key] = prepare_whitelist(patterns) + return wlist whitelists = load_whitelists(os.path.dirname(__file__)) diff --git a/automation/PRESUBMIT.py b/deprecated/automation/PRESUBMIT.py index 56a43e6c..56a43e6c 100644 --- a/automation/PRESUBMIT.py +++ b/deprecated/automation/PRESUBMIT.py diff --git a/automation/__init__.py b/deprecated/automation/__init__.py index 8b137891..8b137891 100644 --- a/automation/__init__.py +++ b/deprecated/automation/__init__.py diff --git a/automation/all_tests.py b/deprecated/automation/all_tests.py index e7b70884..e7b70884 100644 --- a/automation/all_tests.py +++ b/deprecated/automation/all_tests.py diff --git a/automation/clients/__init__.py b/deprecated/automation/clients/__init__.py index 8b137891..8b137891 100644 --- a/automation/clients/__init__.py +++ b/deprecated/automation/clients/__init__.py diff --git a/automation/clients/android.py b/deprecated/automation/clients/android.py index 06e76d29..06e76d29 100755 --- a/automation/clients/android.py +++ b/deprecated/automation/clients/android.py diff --git a/automation/clients/chromeos.py b/deprecated/automation/clients/chromeos.py index 572320fd..572320fd 100755 --- a/automation/clients/chromeos.py +++ b/deprecated/automation/clients/chromeos.py diff --git a/automation/clients/crosstool.py b/deprecated/automation/clients/crosstool.py index 9ba83807..9ba83807 100755 --- a/automation/clients/crosstool.py +++ b/deprecated/automation/clients/crosstool.py diff --git a/automation/clients/dejagnu_compiler.py b/deprecated/automation/clients/dejagnu_compiler.py index 7448b87e..7448b87e 100755 --- a/automation/clients/dejagnu_compiler.py +++ b/deprecated/automation/clients/dejagnu_compiler.py diff --git a/automation/clients/helper/__init__.py b/deprecated/automation/clients/helper/__init__.py index 8b137891..8b137891 100644 --- a/automation/clients/helper/__init__.py +++ b/deprecated/automation/clients/helper/__init__.py diff --git a/automation/clients/helper/android.py b/deprecated/automation/clients/helper/android.py index 7ff2ac1c..7ff2ac1c 100644 --- a/automation/clients/helper/android.py +++ b/deprecated/automation/clients/helper/android.py diff --git a/automation/clients/helper/chromeos.py b/deprecated/automation/clients/helper/chromeos.py index e7157451..e7157451 100644 --- a/automation/clients/helper/chromeos.py +++ b/deprecated/automation/clients/helper/chromeos.py diff --git a/automation/clients/helper/crosstool.py b/deprecated/automation/clients/helper/crosstool.py index 80154b25..80154b25 100644 --- a/automation/clients/helper/crosstool.py +++ b/deprecated/automation/clients/helper/crosstool.py diff --git a/automation/clients/helper/jobs.py b/deprecated/automation/clients/helper/jobs.py index 96a1c408..96a1c408 100644 --- a/automation/clients/helper/jobs.py +++ b/deprecated/automation/clients/helper/jobs.py diff --git a/automation/clients/helper/perforce.py b/deprecated/automation/clients/helper/perforce.py index 1f2dfe79..1f2dfe79 100644 --- a/automation/clients/helper/perforce.py +++ b/deprecated/automation/clients/helper/perforce.py diff --git a/automation/clients/nightly.py b/deprecated/automation/clients/nightly.py index d35c4eca..d35c4eca 100755 --- a/automation/clients/nightly.py +++ b/deprecated/automation/clients/nightly.py diff --git a/automation/clients/output_test.py b/deprecated/automation/clients/output_test.py index 73c26eed..73c26eed 100755 --- a/automation/clients/output_test.py +++ b/deprecated/automation/clients/output_test.py diff --git a/automation/clients/pwd_test.py b/deprecated/automation/clients/pwd_test.py index 493444d5..493444d5 100755 --- a/automation/clients/pwd_test.py +++ b/deprecated/automation/clients/pwd_test.py diff --git a/automation/clients/report/dejagnu.sh b/deprecated/automation/clients/report/dejagnu.sh index fadd8a0c..fadd8a0c 100755 --- a/automation/clients/report/dejagnu.sh +++ b/deprecated/automation/clients/report/dejagnu.sh diff --git a/automation/clients/report/dejagnu/__init__.py b/deprecated/automation/clients/report/dejagnu/__init__.py index 8b137891..8b137891 100644 --- a/automation/clients/report/dejagnu/__init__.py +++ b/deprecated/automation/clients/report/dejagnu/__init__.py diff --git a/automation/clients/report/dejagnu/main.py b/deprecated/automation/clients/report/dejagnu/main.py index 62f095e1..62f095e1 100644 --- a/automation/clients/report/dejagnu/main.py +++ b/deprecated/automation/clients/report/dejagnu/main.py diff --git a/automation/clients/report/dejagnu/manifest.py b/deprecated/automation/clients/report/dejagnu/manifest.py index 5831d1b0..5831d1b0 100644 --- a/automation/clients/report/dejagnu/manifest.py +++ b/deprecated/automation/clients/report/dejagnu/manifest.py diff --git a/automation/clients/report/dejagnu/report.html b/deprecated/automation/clients/report/dejagnu/report.html index 39b39e09..39b39e09 100644 --- a/automation/clients/report/dejagnu/report.html +++ b/deprecated/automation/clients/report/dejagnu/report.html diff --git a/automation/clients/report/dejagnu/report.py b/deprecated/automation/clients/report/dejagnu/report.py index 191a5389..191a5389 100644 --- a/automation/clients/report/dejagnu/report.py +++ b/deprecated/automation/clients/report/dejagnu/report.py diff --git a/automation/clients/report/dejagnu/summary.py b/deprecated/automation/clients/report/dejagnu/summary.py index d573c691..d573c691 100644 --- a/automation/clients/report/dejagnu/summary.py +++ b/deprecated/automation/clients/report/dejagnu/summary.py diff --git a/automation/clients/report/validate_failures.py b/deprecated/automation/clients/report/validate_failures.py index d8776ba5..d8776ba5 100755 --- a/automation/clients/report/validate_failures.py +++ b/deprecated/automation/clients/report/validate_failures.py diff --git a/automation/common/__init__.py b/deprecated/automation/common/__init__.py index 8b137891..8b137891 100644 --- a/automation/common/__init__.py +++ b/deprecated/automation/common/__init__.py diff --git a/automation/common/command.py b/deprecated/automation/common/command.py index c56e9fad..c56e9fad 100644 --- a/automation/common/command.py +++ b/deprecated/automation/common/command.py diff --git a/automation/common/command_executer.py b/deprecated/automation/common/command_executer.py index c0f314f5..c0f314f5 100644 --- a/automation/common/command_executer.py +++ b/deprecated/automation/common/command_executer.py diff --git a/automation/common/command_executer_test.py b/deprecated/automation/common/command_executer_test.py index 2caaa146..2caaa146 100755 --- a/automation/common/command_executer_test.py +++ b/deprecated/automation/common/command_executer_test.py diff --git a/automation/common/events.py b/deprecated/automation/common/events.py index ad3ec844..ad3ec844 100644 --- a/automation/common/events.py +++ b/deprecated/automation/common/events.py diff --git a/automation/common/job.py b/deprecated/automation/common/job.py index e845ab25..e845ab25 100644 --- a/automation/common/job.py +++ b/deprecated/automation/common/job.py diff --git a/automation/common/job_group.py b/deprecated/automation/common/job_group.py index 96912fc1..96912fc1 100644 --- a/automation/common/job_group.py +++ b/deprecated/automation/common/job_group.py diff --git a/automation/common/logger.py b/deprecated/automation/common/logger.py index 4aeee052..4aeee052 100644 --- a/automation/common/logger.py +++ b/deprecated/automation/common/logger.py diff --git a/automation/common/machine.py b/deprecated/automation/common/machine.py index 4db0db0d..4db0db0d 100644 --- a/automation/common/machine.py +++ b/deprecated/automation/common/machine.py diff --git a/automation/common/machine_test.py b/deprecated/automation/common/machine_test.py index f66299f5..f66299f5 100755 --- a/automation/common/machine_test.py +++ b/deprecated/automation/common/machine_test.py diff --git a/automation/common/state_machine.py b/deprecated/automation/common/state_machine.py index d1cf42c8..d1cf42c8 100644 --- a/automation/common/state_machine.py +++ b/deprecated/automation/common/state_machine.py diff --git a/automation/server/__init__.py b/deprecated/automation/server/__init__.py index 8b137891..8b137891 100644 --- a/automation/server/__init__.py +++ b/deprecated/automation/server/__init__.py diff --git a/automation/server/job_executer.py b/deprecated/automation/server/job_executer.py index 30b59463..30b59463 100644 --- a/automation/server/job_executer.py +++ b/deprecated/automation/server/job_executer.py diff --git a/automation/server/job_group_manager.py b/deprecated/automation/server/job_group_manager.py index d66f5e07..d66f5e07 100644 --- a/automation/server/job_group_manager.py +++ b/deprecated/automation/server/job_group_manager.py diff --git a/automation/server/job_manager.py b/deprecated/automation/server/job_manager.py index 7a65b918..7a65b918 100644 --- a/automation/server/job_manager.py +++ b/deprecated/automation/server/job_manager.py diff --git a/automation/server/machine_manager.py b/deprecated/automation/server/machine_manager.py index b7186077..b7186077 100644 --- a/automation/server/machine_manager.py +++ b/deprecated/automation/server/machine_manager.py diff --git a/automation/server/machine_manager_test.py b/deprecated/automation/server/machine_manager_test.py index 2fa5bb4b..2fa5bb4b 100755 --- a/automation/server/machine_manager_test.py +++ b/deprecated/automation/server/machine_manager_test.py diff --git a/automation/server/monitor/__init__.py b/deprecated/automation/server/monitor/__init__.py index 8b137891..8b137891 100644 --- a/automation/server/monitor/__init__.py +++ b/deprecated/automation/server/monitor/__init__.py diff --git a/automation/server/monitor/dashboard.py b/deprecated/automation/server/monitor/dashboard.py index f6befed8..f6befed8 100644 --- a/automation/server/monitor/dashboard.py +++ b/deprecated/automation/server/monitor/dashboard.py diff --git a/automation/server/monitor/manage.py b/deprecated/automation/server/monitor/manage.py index 59f6e216..59f6e216 100755 --- a/automation/server/monitor/manage.py +++ b/deprecated/automation/server/monitor/manage.py diff --git a/automation/server/monitor/settings.py b/deprecated/automation/server/monitor/settings.py index 8cd20e35..8cd20e35 100644 --- a/automation/server/monitor/settings.py +++ b/deprecated/automation/server/monitor/settings.py diff --git a/automation/server/monitor/start.sh b/deprecated/automation/server/monitor/start.sh index 4fc53bef..4fc53bef 100755 --- a/automation/server/monitor/start.sh +++ b/deprecated/automation/server/monitor/start.sh diff --git a/automation/server/monitor/static/style.css b/deprecated/automation/server/monitor/static/style.css index b571b059..b571b059 100644 --- a/automation/server/monitor/static/style.css +++ b/deprecated/automation/server/monitor/static/style.css diff --git a/automation/server/monitor/templates/base.html b/deprecated/automation/server/monitor/templates/base.html index 95ffc222..95ffc222 100644 --- a/automation/server/monitor/templates/base.html +++ b/deprecated/automation/server/monitor/templates/base.html diff --git a/automation/server/monitor/templates/job.html b/deprecated/automation/server/monitor/templates/job.html index 90acd969..90acd969 100644 --- a/automation/server/monitor/templates/job.html +++ b/deprecated/automation/server/monitor/templates/job.html diff --git a/automation/server/monitor/templates/job_group.html b/deprecated/automation/server/monitor/templates/job_group.html index b6ed8ea8..b6ed8ea8 100644 --- a/automation/server/monitor/templates/job_group.html +++ b/deprecated/automation/server/monitor/templates/job_group.html diff --git a/automation/server/monitor/templates/job_group_list.html b/deprecated/automation/server/monitor/templates/job_group_list.html index b82fa730..b82fa730 100644 --- a/automation/server/monitor/templates/job_group_list.html +++ b/deprecated/automation/server/monitor/templates/job_group_list.html diff --git a/automation/server/monitor/templates/job_log.html b/deprecated/automation/server/monitor/templates/job_log.html index 937b21b0..937b21b0 100644 --- a/automation/server/monitor/templates/job_log.html +++ b/deprecated/automation/server/monitor/templates/job_log.html diff --git a/automation/server/monitor/templates/machine_list.html b/deprecated/automation/server/monitor/templates/machine_list.html index f81422d3..f81422d3 100644 --- a/automation/server/monitor/templates/machine_list.html +++ b/deprecated/automation/server/monitor/templates/machine_list.html diff --git a/automation/server/monitor/templates/snippet_attribute_table.html b/deprecated/automation/server/monitor/templates/snippet_attribute_table.html index 24bacc17..24bacc17 100644 --- a/automation/server/monitor/templates/snippet_attribute_table.html +++ b/deprecated/automation/server/monitor/templates/snippet_attribute_table.html diff --git a/automation/server/monitor/templates/snippet_code.html b/deprecated/automation/server/monitor/templates/snippet_code.html index 281754d6..281754d6 100644 --- a/automation/server/monitor/templates/snippet_code.html +++ b/deprecated/automation/server/monitor/templates/snippet_code.html diff --git a/automation/server/monitor/templates/snippet_links.html b/deprecated/automation/server/monitor/templates/snippet_links.html index f19fa6e5..f19fa6e5 100644 --- a/automation/server/monitor/templates/snippet_links.html +++ b/deprecated/automation/server/monitor/templates/snippet_links.html diff --git a/automation/server/monitor/urls.py b/deprecated/automation/server/monitor/urls.py index 1a6b2485..1a6b2485 100644 --- a/automation/server/monitor/urls.py +++ b/deprecated/automation/server/monitor/urls.py diff --git a/automation/server/server.py b/deprecated/automation/server/server.py index c8f22521..c8f22521 100755 --- a/automation/server/server.py +++ b/deprecated/automation/server/server.py diff --git a/automation/server/server_test.py b/deprecated/automation/server/server_test.py index 131ebb3b..131ebb3b 100755 --- a/automation/server/server_test.py +++ b/deprecated/automation/server/server_test.py diff --git a/automation/server/test_pool.csv b/deprecated/automation/server/test_pool.csv index b0700c9b..b0700c9b 100644 --- a/automation/server/test_pool.csv +++ b/deprecated/automation/server/test_pool.csv diff --git a/build-binutils/opts.sh b/deprecated/build-binutils/opts.sh index e69de29b..e69de29b 100644 --- a/build-binutils/opts.sh +++ b/deprecated/build-binutils/opts.sh diff --git a/build-gcc/opts.sh b/deprecated/build-gcc/opts.sh index d1e6fadb..d1e6fadb 100644 --- a/build-gcc/opts.sh +++ b/deprecated/build-gcc/opts.sh diff --git a/build_chrome_browser.py b/deprecated/build_chrome_browser.py index c3b78870..c3b78870 100755 --- a/build_chrome_browser.py +++ b/deprecated/build_chrome_browser.py diff --git a/build_tool.py b/deprecated/build_tool.py index 0c2fff65..0c2fff65 100755 --- a/build_tool.py +++ b/deprecated/build_tool.py diff --git a/crb/autotest_gatherer.py b/deprecated/crb/autotest_gatherer.py index f8f7b43f..f8f7b43f 100644 --- a/crb/autotest_gatherer.py +++ b/deprecated/crb/autotest_gatherer.py diff --git a/crb/autotest_run.py b/deprecated/crb/autotest_run.py index 380c578b..380c578b 100644 --- a/crb/autotest_run.py +++ b/deprecated/crb/autotest_run.py diff --git a/crb/crb_driver.py b/deprecated/crb/crb_driver.py index c6403462..c6403462 100755 --- a/crb/crb_driver.py +++ b/deprecated/crb/crb_driver.py diff --git a/crb/machine_manager_singleton.py b/deprecated/crb/machine_manager_singleton.py index 6ffe9684..6ffe9684 100644 --- a/crb/machine_manager_singleton.py +++ b/deprecated/crb/machine_manager_singleton.py diff --git a/crb/table_formatter.py b/deprecated/crb/table_formatter.py index b8e25d5f..b8e25d5f 100644 --- a/crb/table_formatter.py +++ b/deprecated/crb/table_formatter.py diff --git a/cros_login.py b/deprecated/cros_login.py index 06ff8ff0..06ff8ff0 100755 --- a/cros_login.py +++ b/deprecated/cros_login.py diff --git a/cwp/bartlett/app.yaml b/deprecated/cwp/bartlett/app.yaml index 60010f70..60010f70 100644 --- a/cwp/bartlett/app.yaml +++ b/deprecated/cwp/bartlett/app.yaml diff --git a/cwp/bartlett/server.py b/deprecated/cwp/bartlett/server.py index 8fb7d61e..8fb7d61e 100755 --- a/cwp/bartlett/server.py +++ b/deprecated/cwp/bartlett/server.py diff --git a/cwp/bartlett/static/favicon.ico b/deprecated/cwp/bartlett/static/favicon.ico Binary files differindex 19b58c2e..19b58c2e 100644 --- a/cwp/bartlett/static/favicon.ico +++ b/deprecated/cwp/bartlett/static/favicon.ico diff --git a/cwp/bartlett/test/server_tester.py b/deprecated/cwp/bartlett/test/server_tester.py index 585da43a..585da43a 100644 --- a/cwp/bartlett/test/server_tester.py +++ b/deprecated/cwp/bartlett/test/server_tester.py diff --git a/cwp/bartlett/update_appengine_server b/deprecated/cwp/bartlett/update_appengine_server index f3812057..f3812057 100755 --- a/cwp/bartlett/update_appengine_server +++ b/deprecated/cwp/bartlett/update_appengine_server diff --git a/cwp/demo_pipeline.sh b/deprecated/cwp/demo_pipeline.sh index d45c5c44..d45c5c44 100644 --- a/cwp/demo_pipeline.sh +++ b/deprecated/cwp/demo_pipeline.sh diff --git a/cwp/interpreter/app_engine_pull.py b/deprecated/cwp/interpreter/app_engine_pull.py index d092e2a3..d092e2a3 100644 --- a/cwp/interpreter/app_engine_pull.py +++ b/deprecated/cwp/interpreter/app_engine_pull.py diff --git a/cwp/interpreter/symbolizer.py b/deprecated/cwp/interpreter/symbolizer.py index 4ece480d..4ece480d 100644 --- a/cwp/interpreter/symbolizer.py +++ b/deprecated/cwp/interpreter/symbolizer.py diff --git a/cwp/performance/experiment_gen.py b/deprecated/cwp/performance/experiment_gen.py index a12da2c5..a12da2c5 100644 --- a/cwp/performance/experiment_gen.py +++ b/deprecated/cwp/performance/experiment_gen.py diff --git a/dejagnu/__init__.py b/deprecated/dejagnu/__init__.py index 8b137891..8b137891 100644 --- a/dejagnu/__init__.py +++ b/deprecated/dejagnu/__init__.py diff --git a/dejagnu/boards/chromeos-machine.exp b/deprecated/dejagnu/boards/chromeos-machine.exp index b168b677..b168b677 100644 --- a/dejagnu/boards/chromeos-machine.exp +++ b/deprecated/dejagnu/boards/chromeos-machine.exp diff --git a/dejagnu/boards/gdb.exp.in b/deprecated/dejagnu/boards/gdb.exp.in index e4bbab39..e4bbab39 100644 --- a/dejagnu/boards/gdb.exp.in +++ b/deprecated/dejagnu/boards/gdb.exp.in diff --git a/dejagnu/boards/gdbserver.sh.in b/deprecated/dejagnu/boards/gdbserver.sh.in index b74c685c..b74c685c 100644 --- a/dejagnu/boards/gdbserver.sh.in +++ b/deprecated/dejagnu/boards/gdbserver.sh.in diff --git a/dejagnu/chromeos.exp.in b/deprecated/dejagnu/chromeos.exp.in index 02fc3af2..02fc3af2 100644 --- a/dejagnu/chromeos.exp.in +++ b/deprecated/dejagnu/chromeos.exp.in diff --git a/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi b/deprecated/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi index e94a7905..e94a7905 100644 --- a/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi +++ b/deprecated/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi diff --git a/dejagnu/gdb_baseline/i686-pc-linux-gnu b/deprecated/dejagnu/gdb_baseline/i686-pc-linux-gnu index e3e1ee7c..e3e1ee7c 100644 --- a/dejagnu/gdb_baseline/i686-pc-linux-gnu +++ b/deprecated/dejagnu/gdb_baseline/i686-pc-linux-gnu diff --git a/dejagnu/gdb_baseline/x86_64-cros-linux-gnu b/deprecated/dejagnu/gdb_baseline/x86_64-cros-linux-gnu index a9245eab..a9245eab 100644 --- a/dejagnu/gdb_baseline/x86_64-cros-linux-gnu +++ b/deprecated/dejagnu/gdb_baseline/x86_64-cros-linux-gnu diff --git a/dejagnu/gdb_dejagnu.py b/deprecated/dejagnu/gdb_dejagnu.py index 91fa51fd..91fa51fd 100755 --- a/dejagnu/gdb_dejagnu.py +++ b/deprecated/dejagnu/gdb_dejagnu.py diff --git a/dejagnu/run_dejagnu.py b/deprecated/dejagnu/run_dejagnu.py index 7b6e6fa3..7b6e6fa3 100755 --- a/dejagnu/run_dejagnu.py +++ b/deprecated/dejagnu/run_dejagnu.py diff --git a/dejagnu/site.exp b/deprecated/dejagnu/site.exp index beaa958d..beaa958d 100644 --- a/dejagnu/site.exp +++ b/deprecated/dejagnu/site.exp diff --git a/fdo_scripts/divide_and_merge_profiles.py b/deprecated/fdo_scripts/divide_and_merge_profiles.py index 5108aa6f..5108aa6f 100755 --- a/fdo_scripts/divide_and_merge_profiles.py +++ b/deprecated/fdo_scripts/divide_and_merge_profiles.py diff --git a/fdo_scripts/divide_and_merge_profiles_test.py b/deprecated/fdo_scripts/divide_and_merge_profiles_test.py index 2bfb5cf1..2bfb5cf1 100755 --- a/fdo_scripts/divide_and_merge_profiles_test.py +++ b/deprecated/fdo_scripts/divide_and_merge_profiles_test.py diff --git a/fdo_scripts/profile_cycler.py b/deprecated/fdo_scripts/profile_cycler.py index 176f2d4a..176f2d4a 100755 --- a/fdo_scripts/profile_cycler.py +++ b/deprecated/fdo_scripts/profile_cycler.py diff --git a/fdo_scripts/summarize_hot_blocks.py b/deprecated/fdo_scripts/summarize_hot_blocks.py index 20c07fa4..20c07fa4 100644 --- a/fdo_scripts/summarize_hot_blocks.py +++ b/deprecated/fdo_scripts/summarize_hot_blocks.py diff --git a/fdo_scripts/vanilla_vs_fdo.py b/deprecated/fdo_scripts/vanilla_vs_fdo.py index 6f42839d..6f42839d 100644 --- a/fdo_scripts/vanilla_vs_fdo.py +++ b/deprecated/fdo_scripts/vanilla_vs_fdo.py diff --git a/generate-waterfall-reports.py b/deprecated/generate-waterfall-reports.py index a67cd6ca..a67cd6ca 100755 --- a/generate-waterfall-reports.py +++ b/deprecated/generate-waterfall-reports.py diff --git a/get_common_image_version.py b/deprecated/get_common_image_version.py index 4bb6949f..4bb6949f 100755 --- a/get_common_image_version.py +++ b/deprecated/get_common_image_version.py diff --git a/mem_tests/README.md b/deprecated/mem_tests/README.md index 44bf16c9..44bf16c9 100644 --- a/mem_tests/README.md +++ b/deprecated/mem_tests/README.md diff --git a/mem_tests/clean_data.py b/deprecated/mem_tests/clean_data.py index 1433ba41..1433ba41 100755 --- a/mem_tests/clean_data.py +++ b/deprecated/mem_tests/clean_data.py diff --git a/mem_tests/mem_groups.py b/deprecated/mem_tests/mem_groups.py index 5d593872..5d593872 100755 --- a/mem_tests/mem_groups.py +++ b/deprecated/mem_tests/mem_groups.py diff --git a/mem_tests/total_mem_actual.py b/deprecated/mem_tests/total_mem_actual.py index d2a0cedf..d2a0cedf 100755 --- a/mem_tests/total_mem_actual.py +++ b/deprecated/mem_tests/total_mem_actual.py diff --git a/mem_tests/total_mem_sampled.py b/deprecated/mem_tests/total_mem_sampled.py index 32aa527c..32aa527c 100755 --- a/mem_tests/total_mem_sampled.py +++ b/deprecated/mem_tests/total_mem_sampled.py diff --git a/mem_tests/utils.py b/deprecated/mem_tests/utils.py index 38bd89ca..38bd89ca 100644 --- a/mem_tests/utils.py +++ b/deprecated/mem_tests/utils.py diff --git a/new-generate-waterfall-reports.py b/deprecated/new-generate-waterfall-reports.py index ef48f8be..ef48f8be 100755 --- a/new-generate-waterfall-reports.py +++ b/deprecated/new-generate-waterfall-reports.py diff --git a/produce_output.py b/deprecated/produce_output.py index 46512c49..46512c49 100755 --- a/produce_output.py +++ b/deprecated/produce_output.py diff --git a/remote_gcc_build.py b/deprecated/remote_gcc_build.py index edd0d2b6..edd0d2b6 100755 --- a/remote_gcc_build.py +++ b/deprecated/remote_gcc_build.py diff --git a/repo_to_repo.py b/deprecated/repo_to_repo.py index 91c5d580..91c5d580 100755 --- a/repo_to_repo.py +++ b/deprecated/repo_to_repo.py diff --git a/test_gcc_dejagnu.py b/deprecated/test_gcc_dejagnu.py index 4ea0c51e..4ea0c51e 100755 --- a/test_gcc_dejagnu.py +++ b/deprecated/test_gcc_dejagnu.py diff --git a/test_gdb_dejagnu.py b/deprecated/test_gdb_dejagnu.py index 6f37a4c9..6f37a4c9 100755 --- a/test_gdb_dejagnu.py +++ b/deprecated/test_gdb_dejagnu.py diff --git a/test_toolchains.py b/deprecated/test_toolchains.py index 8684653f..8684653f 100755 --- a/test_toolchains.py +++ b/deprecated/test_toolchains.py diff --git a/verify_compiler.py b/deprecated/verify_compiler.py index b70c1257..b70c1257 100755 --- a/verify_compiler.py +++ b/deprecated/verify_compiler.py diff --git a/weekly_report.py b/deprecated/weekly_report.py index 01db867d..476b9f0d 100755 --- a/weekly_report.py +++ b/deprecated/weekly_report.py @@ -1,6 +1,8 @@ #!/usr/bin/env python2 +# -*- coding: utf-8 -*- # # Copyright Google Inc. 2014 + """Module to generate the 7-day crosperf reports.""" from __future__ import print_function @@ -205,7 +207,7 @@ def Main(argv): '%s.%s' % (timestamp, options.board), 'weekly_tests') for day in WEEKDAYS: - startdir = os.path.join(constants.CROSTC_WORKSPACE, day) + startdir = os.path.join(constants.CROSTC_WORKSPACE, 'nightly-tests', day) num_dirs = os.listdir(startdir) for d in num_dirs: exp_file = os.path.join(startdir, d, 'toolchain_experiment.txt') diff --git a/go/chromeos/setup_chromeos_testing.py b/go/chromeos/setup_chromeos_testing.py index b679ddfc..8b535538 100755 --- a/go/chromeos/setup_chromeos_testing.py +++ b/go/chromeos/setup_chromeos_testing.py @@ -1,7 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- -# -# Copyright 2018 Google Inc. All Rightes Reserved +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Generate board-specific scripts for Go compiler testing.""" from __future__ import print_function @@ -73,7 +75,7 @@ def log(msg): def WriteFile(file_content, file_name): - with open(file_name, 'w') as out_file: + with open(file_name, 'w', encoding='utf-8') as out_file: out_file.write(file_content) @@ -82,7 +84,7 @@ def GenerateGoHelperScripts(ce, arm_board, x86_board, chromeos_root): names = { 'x86_64': x86_board, 'arm64': arm_board, - 'arm32': ("%s32" % arm_board) + 'arm32': ('%s32' % arm_board) } toolchain_dir = os.path.join(chromeos_root, 'src', 'third_party', @@ -93,17 +95,17 @@ def GenerateGoHelperScripts(ce, arm_board, x86_board, chromeos_root): toolchain = CROS_TOOLCHAIN_DATA[k] glibc = GLIBC_DATA[k] - base_file = os.path.join(toolchain_dir, ("go_%s" % name)) + base_file = os.path.join(toolchain_dir, ('go_%s' % name)) base_file_content = BASE_TEMPLATE % (name, arch, arch, toolchain, toolchain, toolchain) WriteFile(base_file_content, base_file) - cmd = "chmod 755 %s" % base_file + cmd = 'chmod 755 %s' % base_file ce.RunCommand(cmd) - exec_file = os.path.join(toolchain_dir, ("go_%s_exec" % name)) + exec_file = os.path.join(toolchain_dir, ('go_%s_exec' % name)) exec_file_content = EXEC_TEMPLATE % (name, arch, glibc, name) WriteFile(exec_file_content, exec_file) - cmd = "chmod 755 %s" % exec_file + cmd = 'chmod 755 %s' % exec_file ce.RunCommand(cmd) return 0 @@ -111,7 +113,7 @@ def GenerateGoHelperScripts(ce, arm_board, x86_board, chromeos_root): def UpdateChrootSshConfig(ce, arm_board, arm_dut, x86_board, x86_dut, chromeos_root): - log("Entering UpdateChrootSshConfig") + log('Entering UpdateChrootSshConfig') # Copy testing_rsa to .ssh and set file protections properly. user = getpass.getuser() ssh_dir = os.path.join(chromeos_root, 'chroot', 'home', user, '.ssh') @@ -133,22 +135,22 @@ def UpdateChrootSshConfig(ce, arm_board, arm_dut, x86_board, x86_dut, print('Cannot find %s; you will need to update testing_rsa by hand.' % src_file) else: - log("testing_rsa exists already.") + log('testing_rsa exists already.') # Save ~/.ssh/config file, if not already done. - config_file = os.path.expanduser("~/.ssh/config") + config_file = os.path.expanduser('~/.ssh/config') saved_config_file = os.path.join( - os.path.expanduser("~/.ssh"), "config.save.go-scripts") + os.path.expanduser('~/.ssh'), 'config.save.go-scripts') if not os.path.exists(saved_config_file): - cmd = "cp %s %s" % (config_file, saved_config_file) + cmd = 'cp %s %s' % (config_file, saved_config_file) ret = ce.RunCommand(cmd) if ret != SUCCESS: - print("Error making save copy of ~/.ssh/config. Exiting...") + print('Error making save copy of ~/.ssh/config. Exiting...') sys.exit(1) # Update ~/.ssh/config file - log("Reading ssh config file") - with open(config_file, "r") as input_file: + log('Reading ssh config file') + with open(config_file, 'r') as input_file: config_lines = input_file.read() x86_host_config = CONFIG_TEMPLATE % (x86_board, x86_dut) @@ -158,7 +160,7 @@ def UpdateChrootSshConfig(ce, arm_board, arm_dut, x86_board, x86_dut, config_lines += x86_host_config config_lines += arm_host_config - log("Writing ~/.ssh/config") + log('Writing ~/.ssh/config') WriteFile(config_lines, config_file) return 0 @@ -170,27 +172,27 @@ def CleanUp(ce, x86_board, arm_board, chromeos_root): names = { 'x86_64': x86_board, 'arm64': arm_board, - 'arm32': ("%s32" % arm_board) + 'arm32': ('%s32' % arm_board) } toolchain_dir = os.path.join(chromeos_root, 'src', 'third_party', 'toolchain-utils', 'go', 'chromeos') for k in keys: name = names[k] - base_file = os.path.join(toolchain_dir, ("go_%s" % name)) - exec_file = os.path.join(toolchain_dir, ("go_%s_exec" % name)) + base_file = os.path.join(toolchain_dir, ('go_%s' % name)) + exec_file = os.path.join(toolchain_dir, ('go_%s_exec' % name)) cmd = ('rm -f %s; rm -f %s' % (base_file, exec_file)) ce.RunCommand(cmd) # Restore saved config_file - config_file = os.path.expanduser("~/.ssh/config") + config_file = os.path.expanduser('~/.ssh/config') saved_config_file = os.path.join( - os.path.expanduser("~/.ssh"), "config.save.go-scripts") + os.path.expanduser('~/.ssh'), 'config.save.go-scripts') if not os.path.exists(saved_config_file): - print("Could not find file: %s; unable to restore ~/.ssh/config ." % + print('Could not find file: %s; unable to restore ~/.ssh/config .' % saved_config_file) else: - cmd = "mv %s %s" % (saved_config_file, config_file) + cmd = 'mv %s %s' % (saved_config_file, config_file) ce.RunCommand(cmd) return 0 @@ -219,7 +221,7 @@ def Main(argv): DEBUG = True if not os.path.exists(options.chromeos_root): - print("Invalid ChromeOS Root: %s" % options.chromeos_root) + print('Invalid ChromeOS Root: %s' % options.chromeos_root) ce = command_executer.GetCommandExecuter() all_good = True @@ -245,6 +247,6 @@ def Main(argv): return 0 -if __name__ == "__main__": +if __name__ == '__main__': val = Main(sys.argv) sys.exit(val) diff --git a/heatmaps/heat_map.py b/heatmaps/heat_map.py index 2fd742d2..a989ab70 100755 --- a/heatmaps/heat_map.py +++ b/heatmaps/heat_map.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2015 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -15,7 +15,7 @@ import sys import tempfile from cros_utils import command_executer -import heatmap_generator +from heatmaps import heatmap_generator def IsARepoRoot(directory): diff --git a/heatmaps/heat_map_test.py b/heatmaps/heat_map_test.py index 21f90d41..ad62cd91 100755 --- a/heatmaps/heat_map_test.py +++ b/heatmaps/heat_map_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2019 The Chromium OS Authors. All rights reserved. @@ -9,14 +9,15 @@ from __future__ import print_function -import mock +import unittest.mock as mock import unittest import os from cros_utils import command_executer -import heat_map +from heatmaps import heat_map +from heatmaps import heatmap_generator def make_heatmap(chromeos_root='/path/to/fake/chromeos_root/', @@ -41,7 +42,7 @@ def fake_generate_perf_report_exception(_): class HeatmapTest(unittest.TestCase): """All of our tests for heat_map.""" - #pylint: disable=protected-access + # pylint: disable=protected-access @mock.patch('shutil.copy2') @mock.patch('tempfile.mkdtemp') def test_EnsureFileInChrootAlreadyInside(self, mock_mkdtemp, mock_copy): @@ -81,7 +82,7 @@ class HeatmapTest(unittest.TestCase): self.assertEqual(heatmap.perf_report, '/fake/chroot/inchroot_path/perf_report.txt') - @mock.patch('heatmap_generator.HeatmapGenerator') + @mock.patch.object(heatmap_generator, 'HeatmapGenerator') def test_GetHeatMap(self, mock_heatmap_generator): heatmap = make_heatmap() heatmap._GetHeatMap(10) @@ -107,7 +108,7 @@ class HeatmapTest(unittest.TestCase): new=fake_generate_perf_report_exception) @mock.patch.object(heat_map.HeatMapProducer, '_GetHeatMap') @mock.patch.object(heat_map.HeatMapProducer, '_RemoveFiles') - @mock.patch('__builtin__.print') + @mock.patch('builtins.print') def test_Run_with_exception(self, mock_print, mock_remove_files, mock_get_heatmap, mock_ensure_file_in_chroot): heatmap = make_heatmap() diff --git a/heatmaps/heatmap_generator.py b/heatmaps/heatmap_generator.py index 42fd6352..0dd6ad28 100644 --- a/heatmaps/heatmap_generator.py +++ b/heatmaps/heatmap_generator.py @@ -13,7 +13,7 @@ performed by another script perf-to-inst-page.sh). It can also analyze the symbol names in hot pages. """ -from __future__ import print_function +from __future__ import division, print_function import bisect import collections @@ -93,10 +93,10 @@ class HeatmapGenerator(object): self.max_addr = 1024 * 1024 * 1024 self.ce = command_executer.GetCommandExecuter(log_level=log_level) self.dir = os.path.dirname(os.path.realpath(__file__)) - with open(perf_report) as f: + with open(perf_report, 'r', encoding='utf-8') as f: self.perf_report_contents = f.readlines() # Write histogram results to a text file, in order to use gnu plot to draw - self.hist_temp_output = open('out.txt', 'w') + self.hist_temp_output = open('out.txt', 'w', encoding='utf-8') self.processes = {} self.deleted_processes = {} self.count = 0 @@ -277,7 +277,7 @@ class HeatmapGenerator(object): if address < self.max_addr: self.count += 1 line = '%d/%d: %d %d' % (pid[0], pid[1], self.count, - address / self.page_size * self.page_size) + address // self.page_size * self.page_size) if self.hugepage: if self.hugepage.start <= address < self.hugepage.end: line += ' hugepage' @@ -321,7 +321,7 @@ class HeatmapGenerator(object): names = [x for x in os.listdir('.') if 'inst-histo' in x and '.txt' in x] hist = {} for n in names: - with open(n) as f: + with open(n, encoding='utf-8') as f: for l in f.readlines(): num, addr = l.strip().split(' ') assert int(addr) not in hist @@ -354,7 +354,7 @@ class HeatmapGenerator(object): if 't' not in symbol_type and 'T' not in symbol_type: # Filter out symbols not in text sections continue - if len(self.symbol_addresses) == 0: + if not self.symbol_addresses: # The first symbol in text sections text_section_start = addr self.symbol_addresses.append(0) @@ -374,9 +374,9 @@ class HeatmapGenerator(object): def _map_addr_to_symbol(self, addr): # Find out the symbol name - assert len(self.symbol_addresses) > 0 + assert self.symbol_addresses index = bisect.bisect(self.symbol_addresses, addr) - assert index > 0 and index <= len(self.symbol_names), \ + assert 0 < index <= len(self.symbol_names), \ 'Failed to find an index (%d) in the list (len=%d)' % ( index, len(self.symbol_names)) return self.symbol_names[index - 1] @@ -387,7 +387,7 @@ class HeatmapGenerator(object): print( '----------------------------------------------------------', file=fp) print( - 'Page Offset: %d MB, Count: %d' % (page_num / 1024 / 1024, + 'Page Offset: %d MB, Count: %d' % (page_num // 1024 // 1024, sample_num), file=fp) @@ -400,8 +400,8 @@ class HeatmapGenerator(object): if pid is None: # The sampling is not on Chrome continue - if addr / self.page_size != ( - self.processes[pid].start_address + page_num) / self.page_size: + if addr // self.page_size != ( + self.processes[pid].start_address + page_num) // self.page_size: # Sampling not in the current page continue @@ -410,14 +410,14 @@ class HeatmapGenerator(object): assert name, 'Failed to find symbol name of addr %x' % addr symbol_counts[name] += 1 - assert sum(symbol_counts.itervalues()) == sample_num, \ + assert sum(symbol_counts.values()) == sample_num, \ 'Symbol name matching missing for some addresses: %d vs %d' % ( - sum(symbol_counts.itervalues()), sample_num) + sum(symbol_counts.values()), sample_num) # Print out the symbol names sorted by the number of samples in # the page for name, count in sorted( - symbol_counts.iteritems(), key=lambda kv: kv[1], reverse=True): + symbol_counts.items(), key=lambda kv: kv[1], reverse=True): if count == 0: break print('> %s : %d' % (name, count), file=fp) @@ -434,20 +434,19 @@ class HeatmapGenerator(object): # Read histogram from histo.txt hist = self._restore_histogram() # Sort the pages in histogram - sorted_hist = sorted( - hist.iteritems(), key=lambda value: value[1], reverse=True) + sorted_hist = sorted(hist.items(), key=lambda value: value[1], reverse=True) # Generate symbolizations self._read_symbols_from_binary(binary) # Write hottest pages - with open('addr2symbol.txt', 'w') as fp: + with open('addr2symbol.txt', 'w', encoding='utf-8') as fp: if self.hugepage: # Print hugepage region first print( 'Hugepage top %d hot pages (%d MB - %d MB):' % - (top_n, self.hugepage.start / 1024 / 1024, - self.hugepage.end / 1024 / 1024), + (top_n, self.hugepage.start // 1024 // 1024, + self.hugepage.end // 1024 // 1024), file=fp) pages_to_print = [(k, v) for k, v in sorted_hist @@ -464,5 +463,5 @@ class HeatmapGenerator(object): self._print_symbols_in_hot_pages(fp, pages_to_print) else: # Print top_n hottest pages. - pages_to_print = [(k, v) for k, v in sorted_hist][:top_n] + pages_to_print = sorted_hist[:top_n] self._print_symbols_in_hot_pages(fp, pages_to_print) diff --git a/heatmaps/heatmap_generator_test.py b/heatmaps/heatmap_generator_test.py index 0c0bbfc8..5008c653 100755 --- a/heatmaps/heatmap_generator_test.py +++ b/heatmaps/heatmap_generator_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,11 +8,12 @@ from __future__ import division, print_function -import mock -import os +import unittest.mock as mock import unittest -import heatmap_generator +import os + +from heatmaps import heatmap_generator def _write_perf_mmap(pid, tid, addr, size, fp): diff --git a/llvm_extra/create_ebuild_file.py b/llvm_extra/create_ebuild_file.py index 459e702a..058a270b 100755 --- a/llvm_extra/create_ebuild_file.py +++ b/llvm_extra/create_ebuild_file.py @@ -1,25 +1,21 @@ -#!/usr/bin/env python2 - +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # Copyright 2018 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function +"""Create llvm ebuild file. -import os -import sys +This script takes an existing host llvm compiler ebuild and +creates another build that should be installable in a prefixed location. +The script patches a few lines in the llvm ebuild to make that happen. -# This script takes an existing host llvm compiler ebuild and -# creates another build that should be installable in a prefixed location. -# The script patches a few lines in the llvm ebuild to make that happen. -# -# Since the script is based on the current llvm ebuild patterns, -# it may need to be updated if those patterns change. -# -# This script should normally be invoked by the shell script -# create_llvm_extra.sh . +Since the script is based on the current llvm ebuild patterns, +it may need to be updated if those patterns change. + +This script should normally be invoked by the shell script +create_llvm_extra.sh . -""" Below is an example of the expected diff of the newly generated ebuild with some explanation of the diffs. @@ -40,8 +36,8 @@ diff -Nuar llvm-pre7.0_pre335547_p20180529.ebuild newly-created-file.ebuild # Change USE flags to match llvm ebuild installtion. To see the set of flags enabled in llvm compiler ebuild, run $ sudo emerge -pv llvm --IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi multitarget -+IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi +multitarget +-IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi" ++IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi ncurses ocaml python llvm-next llvm-tot test xml video_cards_radeon" COMMON_DEPEND=" @@ -59,8 +55,8 @@ diff -Nuar llvm-pre7.0_pre335547_p20180529.ebuild newly-created-file.ebuild # Allow custom cmake build types (like 'Gentoo') # Convert use of PN to llvm in epatch commands. -- epatch "${FILESDIR}"/cmake/${PN}-3.8-allow_custom_cmake_build_types.patch -+ epatch "${FILESDIR}"/cmake/llvm-3.8-allow_custom_cmake_build_types.patch +- epatch "${FILESDIR}"/cmake/${PN}-3.8-allow_custom_cmake_build.patch ++ epatch "${FILESDIR}"/cmake/llvm-3.8-allow_custom_cmake_build.patch # crbug/591436 epatch "${FILESDIR}"/clang-executable-detection.patch @@ -94,6 +90,12 @@ diff -Nuar llvm-pre7.0_pre335547_p20180529.ebuild newly-created-file.ebuild # some users may find it useful """ +from __future__ import print_function + +import os +import sys + + def process_line(line, text): # Process the line and append to the text we want to generate. # Check if line has any patterns that we want to handle. @@ -103,7 +105,7 @@ def process_line(line, text): text.append(line) elif line.startswith('SLOT='): # Change SLOT to "${PV%%_p[[:digit:]]*}" - SLOT_STRING='SLOT="${PV%%_p[[:digit:]]*}"\n' + SLOT_STRING = 'SLOT="${PV%%_p[[:digit:]]*}"\n' text.append(SLOT_STRING) elif line.startswith('IUSE') and 'multitarget' in line: # Enable multitarget USE flag. @@ -137,9 +139,9 @@ def process_line(line, text): def main(): if len(sys.argv) != 3: - filename = os.path.basename(__file__) - print ('Usage: ', filename,' <input.ebuild> <output.ebuild>') - return 1 + filename = os.path.basename(__file__) + print('Usage: ', filename, ' <input.ebuild> <output.ebuild>') + return 1 text = [] with open(sys.argv[1], 'r') as infile: @@ -147,10 +149,10 @@ def main(): process_line(line, text) with open(sys.argv[2], 'w') as outfile: - outfile.write("".join(text)) + outfile.write(''.join(text)) return 0 -if __name__== "__main__": +if __name__ == '__main__': sys.exit(main()) diff --git a/llvm_tools/custom_script_example.py b/llvm_tools/custom_script_example.py index 7e107ad8..38dff007 100755 --- a/llvm_tools/custom_script_example.py +++ b/llvm_tools/custom_script_example.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/lock_machine.py b/lock_machine.py index 244edfb9..06c0e80c 100755 --- a/lock_machine.py +++ b/lock_machine.py @@ -59,9 +59,9 @@ class LockManager(object): # set it long enough to cover the period to finish nightly rotation tests. LEASE_MINS = 1440 - SKYLAB_CREDENTIAL = '/usr/local/google/home/mobiletc-prebuild' \ - '/sheriff_utils/skylab_credential' \ - '/chromeos-swarming-credential.json' + SKYLAB_CREDENTIAL = ('/usr/local/google/home/mobiletc-prebuild' + '/sheriff_utils/credentials/skylab' + '/chromeos-swarming-credential.json') SWARMING = 'chromite/third_party/swarming.client/swarming.py' SUCCESS = 0 diff --git a/orderfile/post_process_orderfile.py b/orderfile/post_process_orderfile.py index e24ab1cd..3db0b3b8 100755 --- a/orderfile/post_process_orderfile.py +++ b/orderfile/post_process_orderfile.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -79,7 +79,7 @@ def main(argv): options = parser.parse_args(argv) if not os.path.exists(options.input_file): - sys.exit('Input orderfile doesn\'t exist.') + sys.exit("Input orderfile doesn\'t exist.") with open(options.input_file) as in_stream, \ open(options.chrome_nm) as chrome_nm_stream, \ diff --git a/orderfile/post_process_orderfile_test.py b/orderfile/post_process_orderfile_test.py index 2532b8b3..a5fb2c73 100755 --- a/orderfile/post_process_orderfile_test.py +++ b/orderfile/post_process_orderfile_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -41,7 +41,7 @@ def _cleanup(files): class Tests(unittest.TestCase): """All of our tests for post_process_orderfile.""" - #pylint: disable=protected-access + # pylint: disable=protected-access def test__parse_nm_output(self): temp_dir = tempfile.mkdtemp() self.addCleanup(_cleanup, [temp_dir]) diff --git a/pgo_tools/merge_profdata_and_upload.py b/pgo_tools/merge_profdata_and_upload.py index dddc7f1e..ea95289b 100755 --- a/pgo_tools/merge_profdata_and_upload.py +++ b/pgo_tools/merge_profdata_and_upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -56,7 +56,8 @@ _LLVMMetadata = collections.namedtuple('_LLVMMetadata', ['head_sha']) def _get_gs_latest(remote_lastest): assert remote_lastest.startswith(_GS_PREFIX) try: - return subprocess.check_output(['gsutil', 'cat', remote_lastest]) + return subprocess.check_output(['gsutil', 'cat', remote_lastest], + encoding='utf-8') except subprocess.CalledProcessError: raise RuntimeError('Lastest artifacts not found: %s' % remote_lastest) @@ -142,7 +143,8 @@ def _tar_and_upload_profdata(profdata, name_suffix): print('Uploading tarball to gs.\nCMD: %s\n' % upload_cmd) # gsutil prints all status to stderr, oddly enough. - gs_output = subprocess.check_output(upload_cmd, stderr=subprocess.STDOUT) + gs_output = subprocess.check_output( + upload_cmd, stderr=subprocess.STDOUT, encoding='utf-8') print(gs_output) # gsutil exits successfully even if it uploaded nothing. It prints a summary @@ -227,7 +229,7 @@ def main(): for tryjob in args.tryjob: fetch_and_append_artifacts(tryjob) - assert heads, 'Didn\'t fetch anything?' + assert heads, "Didn't fetch anything?" def die_with_head_complaint(complaint): extra = ' (HEADs found: %s)' % sorted(heads) @@ -239,7 +241,7 @@ def main(): die_with_head_complaint( '%d LLVM HEADs were found, which is more than one. You probably ' 'want a consistent set of HEADs for a profile. If you know you ' - 'don\'t, please specify --llvm_hash, and note that *all* profiles ' + "don't, please specify --llvm_hash, and note that *all* profiles " 'will be merged into this final profile, regardless of their ' 'reported HEAD.' % len(heads)) llvm_hash, = heads @@ -247,7 +249,7 @@ def main(): if llvm_hash not in heads: assert llvm_hash == args.llvm_hash die_with_head_complaint( - 'HEAD %s wasn\'t found in any fetched artifacts.' % llvm_hash) + "HEAD %s wasn't found in any fetched artifacts." % llvm_hash) print('Using LLVM hash: %s' % llvm_hash) diff --git a/remote_kill_test.py b/remote_kill_test.py deleted file mode 100755 index e0f29d0c..00000000 --- a/remote_kill_test.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. -"""Script to wrap test_that script. - -Run this script and kill it. Then run ps -ef to see if sleep -is still running,. -""" - -from __future__ import print_function - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -import argparse -import os -import sys - -from cros_utils import command_executer - - -def Usage(parser, message): - print('ERROR: %s' % message) - parser.print_help() - sys.exit(0) - - -def Main(argv): - parser = argparse.ArgumentParser() - parser.add_argument( - '-c', - '--chromeos_root', - dest='chromeos_root', - help='ChromeOS root checkout directory') - parser.add_argument( - '-r', '--remote', dest='remote', help='Remote chromeos device.') - - _ = parser.parse_args(argv) - ce = command_executer.GetCommandExecuter() - ce.RunCommand('ls; sleep 10000', machine=os.uname()[1]) - return 0 - - -if __name__ == '__main__': - Main(sys.argv[1:]) diff --git a/remote_test.py b/remote_test.py index 62598d5a..98ff62a5 100755 --- a/remote_test.py +++ b/remote_test.py @@ -1,6 +1,10 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # -# Copyright 2010 Google Inc. All Rights Reserved. +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Script to wrap test_that script. This script can login to the chromeos machine using the test private key. diff --git a/run_tests_for.py b/run_tests_for.py index 6f77b12c..19f81722 100755 --- a/run_tests_for.py +++ b/run_tests_for.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2019 The Chromium OS Authors. All rights reserved. @@ -37,6 +37,12 @@ import sys TestSpec = collections.namedtuple('TestSpec', ['directory', 'command']) +# List of python scripts that are not test with relative path to +# toolchain-utils. +non_test_py_files = { + 'debug_info_test/debug_info_test.py', +} + def _make_relative_to_toolchain_utils(toolchain_utils, path): """Cleans & makes a path relative to toolchain_utils. @@ -52,13 +58,26 @@ def _make_relative_to_toolchain_utils(toolchain_utils, path): return result -def _gather_python_tests_in(subdir): +def _filter_python_tests(test_files, toolchain_utils): + """Returns all files that are real python tests.""" + python_tests = [] + for test_file in test_files: + rel_path = _make_relative_to_toolchain_utils(toolchain_utils, test_file) + if rel_path not in non_test_py_files: + python_tests.append(_python_test_to_spec(test_file)) + else: + print('## %s ... NON_TEST_PY_FILE' % rel_path) + return python_tests + + +def _gather_python_tests_in(rel_subdir, toolchain_utils): """Returns all files that appear to be Python tests in a given directory.""" + subdir = os.path.join(toolchain_utils, rel_subdir) test_files = ( os.path.join(subdir, file_name) for file_name in os.listdir(subdir) if file_name.endswith('_test.py') or file_name.endswith('_unittest.py')) - return [_python_test_to_spec(test_file) for test_file in test_files] + return _filter_python_tests(test_files, toolchain_utils) def _run_test(test_spec): @@ -68,7 +87,8 @@ def _run_test(test_spec): cwd=test_spec.directory, stdin=open('/dev/null'), stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') stdout, _ = p.communicate() exit_code = p.wait() return exit_code, stdout @@ -84,13 +104,13 @@ def _python_test_to_spec(test_file): if os.access(test_file, os.X_OK): command = ['./' + file_name] else: - # Assume the user wanted py2. - command = ['python2', file_name] + # Assume the user wanted py3. + command = ['python3', file_name] return TestSpec(directory=test_directory, command=command) -def _autodetect_python_tests_for(test_file): +def _autodetect_python_tests_for(test_file, toolchain_utils): """Given a test file, detect if there may be related tests.""" if not test_file.endswith('.py'): return [] @@ -102,8 +122,7 @@ def _autodetect_python_tests_for(test_file): base = test_file[:-3] candidates = (base + x for x in test_suffixes) test_files = (x for x in candidates if os.path.exists(x)) - - return [_python_test_to_spec(test_file) for test_file in test_files] + return _filter_python_tests(test_files, toolchain_utils) def _run_test_scripts(all_tests, show_successful_output=False): @@ -194,7 +213,7 @@ def _find_forced_subdir_python_tests(test_paths, toolchain_utils): results = [] for d in sorted(gather_test_dirs): - results += _gather_python_tests_in(os.path.join(toolchain_utils, d)) + results += _gather_python_tests_in(d, toolchain_utils) return results @@ -240,7 +259,7 @@ def main(argv): tests_to_run = _find_forced_subdir_python_tests(modified_files, toolchain_utils) for f in modified_files: - tests_to_run += _autodetect_python_tests_for(f) + tests_to_run += _autodetect_python_tests_for(f, toolchain_utils) tests_to_run += _find_go_tests(modified_files) # TestSpecs have lists, so we can't use a set. We'd likely want to keep them diff --git a/setup_chromeos.py b/setup_chromeos.py index 0b51d830..63206492 100755 --- a/setup_chromeos.py +++ b/setup_chromeos.py @@ -1,8 +1,9 @@ -#!/usr/bin/env python2 -# +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # Copyright 2010 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Script to checkout the ChromeOS source. This script sets up the ChromeOS source in the given directory, matching a @@ -85,7 +86,7 @@ def GetVersionSpecFile(version, versions_git): def TimeToCommonVersion(timestamp): """Convert timestamp to common image version.""" tdt = datetime.fromtimestamp(float(timestamp)) - with open(COMMON_VERSIONS, 'r') as f: + with open(COMMON_VERSIONS, 'r', encoding='utf-8') as f: common_list = pickle.load(f) for sv in common_list: sdt = datetime.strptime(sv['date'], '%Y-%m-%d %H:%M:%S.%f') @@ -157,7 +158,7 @@ Default is 'latest_lkgm'.""") if version not in ('lkgm', 'common'): parser.print_help() logger.GetLogger().LogFatal('timestamp option only applies for ' - "versions \"lkgm\" or \"common\"") + 'versions "lkgm" or "common"') if not options.directory: parser.print_help() @@ -181,8 +182,8 @@ Default is 'latest_lkgm'.""") manifests = manifest_versions.ManifestVersions() version = manifests.TimeToVersionChromeOS(time.mktime(time.gmtime())) version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput('found version %s.%s for latest LKGM' % - (version, manifest)) + logger.GetLogger().LogOutput( + 'found version %s.%s for latest LKGM' % (version, manifest)) init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % (versions_repo, version, manifest)) del manifests @@ -193,17 +194,16 @@ Default is 'latest_lkgm'.""") manifests = manifest_versions.ManifestVersions() version = manifests.TimeToVersion(timestamp) version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for LKGM at timestamp %s' % (version, manifest, - timestamp)) + logger.GetLogger().LogOutput('found version %s.%s for LKGM at timestamp %s' + % (version, manifest, timestamp)) init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % (versions_repo, version, manifest)) del manifests elif version == 'latest_common': version = TimeToCommonVersion(time.mktime(time.gmtime())) version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput('found version %s.%s for latest Common image' % - (version, manifest)) + logger.GetLogger().LogOutput( + 'found version %s.%s for latest Common image' % (version, manifest)) init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, manifest)) elif version == 'common': @@ -212,9 +212,9 @@ Default is 'latest_lkgm'.""") logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') version = TimeToCommonVersion(timestamp) version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput('found version %s.%s for latest common image ' - 'at timestamp %s' % (version, manifest, - timestamp)) + logger.GetLogger().LogOutput( + 'found version %s.%s for latest common image ' + 'at timestamp %s' % (version, manifest, timestamp)) init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, manifest)) else: diff --git a/tc_enter_chroot.py b/tc_enter_chroot.py index d919c96d..3a7538ad 100755 --- a/tc_enter_chroot.py +++ b/tc_enter_chroot.py @@ -1,6 +1,9 @@ -#!/usr/bin/env python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Script to enter the ChromeOS chroot with mounted sources. This script enters the chroot with mounted sources. @@ -191,10 +194,10 @@ def Main(argv, return_output=False): '%s/../../../third_party' % os.path.dirname(__file__)) if os.path.isdir(third_party_dir): - mount_point = MountPoint(third_party_dir, - ('%s/%s' % (full_mounted_tc_root, - os.path.basename(third_party_dir))), - getpass.getuser()) + mount_point = MountPoint( + third_party_dir, + ('%s/%s' % (full_mounted_tc_root, os.path.basename(third_party_dir))), + getpass.getuser()) mount_points.append(mount_point) output = options.output @@ -243,16 +246,15 @@ def Main(argv, return_output=False): inner_command = inner_command[3:] command_file = 'tc_enter_chroot.cmd' command_file_path = chromeos_root + '/src/scripts/' + command_file - retv = command_executer.GetCommandExecuter().RunCommand( - 'sudo rm -f ' + command_file_path) + retv = command_executer.GetCommandExecuter().RunCommand('sudo rm -f ' + + command_file_path) if retv != 0: return retv - f = open(command_file_path, 'w') - f.write(inner_command) - f.close() + with open(command_file_path, 'w', encoding='utf-8') as f: + f.write(inner_command) logger.GetLogger().LogCmd(inner_command) - retv = command_executer.GetCommandExecuter().RunCommand( - 'chmod +x ' + command_file_path) + retv = command_executer.GetCommandExecuter().RunCommand('chmod +x ' + + command_file_path) if retv != 0: return retv diff --git a/toolchain_utils_githooks/check-presubmit.py b/toolchain_utils_githooks/check-presubmit.py index 2fea102a..fc6ec9fc 100755 --- a/toolchain_utils_githooks/check-presubmit.py +++ b/toolchain_utils_githooks/check-presubmit.py @@ -226,7 +226,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files): # pylint+golint. def try_run_cros_lint(cros_binary): exit_code, output = run_command_unchecked( - [cros_binary, 'lint', '--'] + files, + [cros_binary, 'lint', '--py3', '--'] + files, toolchain_utils_root, env=fixed_env) diff --git a/update_telemetry_defaults.py b/update_telemetry_defaults.py index 943dc261..c070aeb1 100755 --- a/update_telemetry_defaults.py +++ b/update_telemetry_defaults.py @@ -1,6 +1,9 @@ -#!/usr/bin/env python2 -# -# Copyright 2013 Google Inc. All Rights Reserved. +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Script to maintain the Telemetry benchmark default results file. This script allows the user to see and update the set of default @@ -36,20 +39,20 @@ class TelemetryDefaults(object): def ReadDefaultsFile(self): if os.path.exists(self._filename): - with open(self._filename, 'r') as fp: + with open(self._filename, 'r', encoding='utf-8') as fp: self._defaults = json.load(fp) def WriteDefaultsFile(self): - with open(self._filename, 'w') as fp: + with open(self._filename, 'w', encoding='utf-8') as fp: json.dump(self._defaults, fp, indent=2) - def ListCurrentDefaults(self, benchmark=all): + def ListCurrentDefaults(self, benchmark='all'): # Show user current defaults. By default, show all. The user # can specify the name of a particular benchmark to see only that # benchmark's default values. if len(self._defaults) == 0: print('The benchmark default results are currently empty.') - if benchmark == all: + if benchmark == 'all': for b in self._defaults.keys(): results = self._defaults[b] out_str = b + ' : ' @@ -83,8 +86,8 @@ class TelemetryDefaults(object): print("Updated results set for '%s': " % benchmark) print('%s : %s' % (benchmark, repr(self._defaults[benchmark]))) else: - print("'%s' is not in '%s's default results list." % (result, - benchmark)) + print( + "'%s' is not in '%s's default results list." % (result, benchmark)) else: print("Cannot find benchmark named '%s'" % benchmark) @@ -145,14 +148,14 @@ lower) does not matter, for the command (case of the result name DOES matter): words = inp.split(' ') option = words[0] option = option.lower() - if option == 'h' or option == 'help': + if option in ('h', 'help'): self.ShowOptions() - elif option == 'l' or option == 'list': + elif option in ('l', 'list'): if len(words) == 1: self.ListCurrentDefaults() else: self.ListCurrentDefaults(benchmark=words[1]) - elif option == 'a' or option == 'add': + elif option in ('a', 'add'): if len(words) < 3: self.UsageError(inp) else: @@ -160,31 +163,30 @@ lower) does not matter, for the command (case of the result name DOES matter): resultList = words[2:] for r in resultList: self.AddDefault(benchmark, r) - elif option == 'd' or option == 'delete': + elif option in ('d', 'delete'): if len(words) != 3: self.UsageError(inp) else: benchmark = words[1] result = words[2] self.RemoveDefault(benchmark, result) - elif option == 'r' or option == 'remove': + elif option in ('r', 'remove'): if len(words) != 2: self.UsageError(inp) else: benchmark = words[1] self.RemoveBenchmark(benchmark) - elif option == 'm' or option == 'move': + elif option in ('m', 'move'): if len(words) != 3: self.UsageError(inp) else: old_name = words[1] new_name = words[2] self.RenameBenchmark(old_name, new_name) - elif option == 'q' or option == 'quit': + elif option in ('q', 'quit'): self.WriteDefaultsFile() - return (option == 'q' or option == 'quit' or option == 't' or - option == 'terminate') + return option in ('q', 'quit', 't', 'terminate') def Main(): |