diff options
author | Chih-Hung Hsieh <chh@google.com> | 2021-10-05 23:14:04 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-10-05 23:14:04 +0000 |
commit | f103b9f78baa21b4edf25d5dd5a1226b6c0ff159 (patch) | |
tree | 0cbf6f6e50bf2d7f1d6021a09db50dc7e6dd9ff6 | |
parent | 75d0ad1e93dfec40618bcec58797c8cb896b8198 (diff) | |
parent | 929241978e4eedb4accab7e83ab383d06c3b4621 (diff) | |
download | toolchain-utils-f103b9f78baa21b4edf25d5dd5a1226b6c0ff159.tar.gz |
Merging 9 commit(s) from Chromium's toolchain-utils am: 3134544d54 am: e0c2630c64 am: 929241978e
Original change: https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/1845270
Change-Id: I497d318091cd0bb5742068f99bd7d3998962e1f0
-rw-r--r-- | afdo_metadata/kernel_afdo.json | 11 | ||||
-rwxr-xr-x | afdo_tools/update_kernel_afdo | 13 | ||||
-rw-r--r-- | compiler_wrapper/compiler_wrapper.go | 37 | ||||
-rwxr-xr-x | cwp/cr-os/fetch_gn_descs.py | 1 | ||||
-rwxr-xr-x | llvm_tools/bisect_clang_crashes.py | 2 | ||||
-rwxr-xr-x | llvm_tools/bisect_clang_crashes_unittest.py | 1 | ||||
-rwxr-xr-x | llvm_tools/fetch_cros_sdk_rolls.py | 2 | ||||
-rwxr-xr-x | llvm_tools/nightly_revert_checker.py | 2 | ||||
-rwxr-xr-x | llvm_tools/upload_lexan_crashes_to_forcey.py | 2 | ||||
-rwxr-xr-x | pgo_tools/monitor_pgo_profiles.py | 2 | ||||
-rwxr-xr-x | rust_tools/rust_uprev.py | 135 | ||||
-rwxr-xr-x | rust_tools/rust_uprev_test.py | 61 | ||||
-rwxr-xr-x | rust_tools/rust_watch.py | 2 | ||||
-rwxr-xr-x | rust_tools/rust_watch_test.py | 2 |
14 files changed, 225 insertions, 48 deletions
diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 3529ebb4..a4044c67 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,17 +1,14 @@ { - "chromeos-kernel-3_18": { - "name": "R94-14092.5-1627896981" - }, "chromeos-kernel-4_4": { - "name": "R94-14119.0-1627896842" + "name": "R96-14209.0-1632130382" }, "chromeos-kernel-4_14": { - "name": "R94-14119.0-1627896837" + "name": "R96-14221.0-1632130642" }, "chromeos-kernel-4_19": { - "name": "R94-14119.0-1627896728" + "name": "R96-14209.0-1632130511" }, "chromeos-kernel-5_4": { - "name": "R94-14119.0-1627896791" + "name": "R96-14209.0-1632130436" } } diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index 4b41053f..ff0ab224 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -25,8 +25,10 @@ set -eu set -o pipefail GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel -KVERS="3.18 4.4 4.14 4.19 5.4" +KVERS="4.4 4.14 4.19 5.4" failed_channels="" +# Add skipped chrome branches in ascending order here. +SKIPPED_BRANCHES="95" script_dir=$(dirname "$0") tc_utils_dir="${script_dir}/.." @@ -57,8 +59,14 @@ branch["canary"]=${canary_ref##*/} # Get current branch numbers (number which goes after R). branch_number["stable"]=$(echo "${branch["stable"]}" | \ sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") -branch_number["beta"]="$((branch_number[stable] + 1))" +branch_number["beta"]=$(echo "${branch["beta"]}" | \ + sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") branch_number["canary"]="$((branch_number[beta] + 1))" +for skipped_branch in $SKIPPED_BRANCHES ; do + if [[ ${branch_number["canary"]} == $skipped_branch ]] ; then + ((branch_number[canary]++)) + fi +done # Without arguments the script updates all branches. channels=${1:-"all"} @@ -180,7 +188,6 @@ EOT canary ) commit_contents="afdo_metadata: Publish the new kernel profiles -Update chromeos-kernel-3_18 Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go index 7d949d34..986eabab 100644 --- a/compiler_wrapper/compiler_wrapper.go +++ b/compiler_wrapper/compiler_wrapper.go @@ -74,25 +74,36 @@ func runAndroidClangTidy(env env, cmd *command) error { if err != nil || seconds == 0 { return env.exec(cmd) } + getSourceFile := func() string { + // Note: This depends on Android build system's clang-tidy command line format. + // Last non-flag before "--" in cmd.Args is used as the source file name. + sourceFile := "unknown_file" + for _, arg := range cmd.Args { + if arg == "--" { + break + } + if strings.HasPrefix(arg, "-") { + continue + } + sourceFile = arg + } + return sourceFile + } + startTime := time.Now() err = env.runWithTimeout(cmd, time.Duration(seconds)*time.Second) if !errors.Is(err, context.DeadlineExceeded) { + // When used time is over half of TIDY_TIMEOUT, give a warning. + // These warnings allow users to fix slow jobs before they get worse. + usedSeconds := int(time.Now().Sub(startTime) / time.Second) + if usedSeconds > seconds/2 { + warning := "%s:1:1: warning: clang-tidy used %d seconds.\n" + fmt.Fprintf(env.stdout(), warning, getSourceFile(), usedSeconds) + } return err } // When DeadllineExceeded, print warning messages. - // Note: This depends on Android build system's clang-tidy command line format. - // Last non-flag before "--" in cmd.Args is used as the source file name. - sourceFile := "unknown_file" - for _, arg := range cmd.Args { - if arg == "--" { - break - } - if strings.HasPrefix(arg, "-") { - continue - } - sourceFile = arg - } warning := "%s:1:1: warning: clang-tidy aborted after %d seconds.\n" - fmt.Fprintf(env.stdout(), warning, sourceFile, seconds) + fmt.Fprintf(env.stdout(), warning, getSourceFile(), seconds) fmt.Fprintf(env.stdout(), "TIMEOUT: %s %s\n", cmd.Path, strings.Join(cmd.Args, " ")) // Do not stop Android build. Just give a warning and return no error. return nil diff --git a/cwp/cr-os/fetch_gn_descs.py b/cwp/cr-os/fetch_gn_descs.py index 60d331c5..8a0b2e4e 100755 --- a/cwp/cr-os/fetch_gn_descs.py +++ b/cwp/cr-os/fetch_gn_descs.py @@ -23,7 +23,6 @@ from __future__ import print_function import argparse import json -# pylint: disable=cros-logging-import import logging import os import subprocess diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py index e8ee2ab6..c53db179 100755 --- a/llvm_tools/bisect_clang_crashes.py +++ b/llvm_tools/bisect_clang_crashes.py @@ -7,8 +7,6 @@ """Fetches and submits the artifacts from Chrome OS toolchain's crash bucket. """ -# pylint: disable=cros-logging-import - import argparse import glob import json diff --git a/llvm_tools/bisect_clang_crashes_unittest.py b/llvm_tools/bisect_clang_crashes_unittest.py index c9143450..a3dc0c6d 100755 --- a/llvm_tools/bisect_clang_crashes_unittest.py +++ b/llvm_tools/bisect_clang_crashes_unittest.py @@ -6,7 +6,6 @@ """Tests for bisect_clang_crashes.""" -# pylint: disable=cros-logging-import import glob import logging import os.path diff --git a/llvm_tools/fetch_cros_sdk_rolls.py b/llvm_tools/fetch_cros_sdk_rolls.py index 42af678a..83d7025a 100755 --- a/llvm_tools/fetch_cros_sdk_rolls.py +++ b/llvm_tools/fetch_cros_sdk_rolls.py @@ -10,8 +10,6 @@ gs://chromiumos-sdk/. The hope is that it'll help answer the question "when did the toolchain ebuild ${x} go live?" """ -# pylint: disable=cros-logging-import - import argparse import json import logging diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py index 1a7bc793..6941f3d6 100755 --- a/llvm_tools/nightly_revert_checker.py +++ b/llvm_tools/nightly_revert_checker.py @@ -10,8 +10,6 @@ If any reverts are found that were previously unknown, this cherry-picks them or fires off an email. All LLVM SHAs to monitor are autodetected. """ -# pylint: disable=cros-logging-import - from __future__ import print_function import argparse diff --git a/llvm_tools/upload_lexan_crashes_to_forcey.py b/llvm_tools/upload_lexan_crashes_to_forcey.py index 486fe664..61bf6b7d 100755 --- a/llvm_tools/upload_lexan_crashes_to_forcey.py +++ b/llvm_tools/upload_lexan_crashes_to_forcey.py @@ -6,8 +6,6 @@ """Fetches and submits the latest test-cases from Lexan's crash bucket.""" -# pylint: disable=cros-logging-import - import argparse import contextlib import datetime diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py index cb41eb88..be159b94 100755 --- a/pgo_tools/monitor_pgo_profiles.py +++ b/pgo_tools/monitor_pgo_profiles.py @@ -5,8 +5,6 @@ """Emails the mage if PGO profile generation hasn't succeeded recently.""" -# pylint: disable=cros-logging-import - import argparse import datetime import sys diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py index 16fb0da8..011639df 100755 --- a/rust_tools/rust_uprev.py +++ b/rust_tools/rust_uprev.py @@ -46,6 +46,10 @@ from pathlib import Path from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple from llvm_tools import chroot, git + +EQUERY = 'equery' +GSUTIL = 'gsutil.py' +MIRROR_PATH = 'gs://chromeos-localmirror/distfiles' RUST_PATH = Path( '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust') @@ -55,6 +59,15 @@ def get_command_output(command: List[str], *args, **kwargs) -> str: **kwargs).strip() +def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str: + return subprocess.run(command, + check=False, + stdout=subprocess.PIPE, + encoding='utf-8', + *args, + **kwargs).stdout.strip() + + class RustVersion(NamedTuple): """NamedTuple represents a Rust version""" major: int @@ -90,6 +103,19 @@ class RustVersion(NamedTuple): int(m.group('patch'))) +def compute_rustc_src_name(version: RustVersion) -> str: + return f'rustc-{version}-src.tar.gz' + + +def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str: + return f'rust-bootstrap-{version}.tbz2' + + +def find_ebuild_for_package(name: str) -> os.PathLike: + """Returns the path to the ebuild for the named package.""" + return get_command_output([EQUERY, 'w', name]) + + def find_ebuild_path(directory: Path, name: str, version: Optional[RustVersion] = None) -> Path: @@ -252,7 +278,7 @@ def parse_commandline_args() -> argparse.Namespace: def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] ) -> Optional[Tuple[RustVersion, str, RustVersion]]: if template is None: - ebuild_path = get_command_output(['equery', 'w', 'rust']) + ebuild_path = find_ebuild_for_package('rust') ebuild_name = os.path.basename(ebuild_path) template_version = RustVersion.parse_from_ebuild(ebuild_name) else: @@ -354,14 +380,103 @@ def flip_mirror_in_ebuild(ebuild_file: Path, add: bool) -> None: def ebuild_actions(package: str, actions: List[str], sudo: bool = False) -> None: - ebuild_path_inchroot = get_command_output(['equery', 'w', package]) + ebuild_path_inchroot = find_ebuild_for_package(package) cmd = ['ebuild', ebuild_path_inchroot] + actions if sudo: cmd = ['sudo'] + cmd subprocess.check_call(cmd) +def fetch_distfile_from_mirror(name: str) -> None: + """Gets the named file from the local mirror. + + This ensures that the file exists on the mirror, and + that we can read it. We overwrite any existing distfile + to ensure the checksums that update_manifest() records + match the file as it exists on the mirror. + + This function also attempts to verify the ACL for + the file (which is expected to have READER permission + for allUsers). We can only see the ACL if the user + gsutil runs with is the owner of the file. If not, + we get an access denied error. We also count this + as a success, because it means we were able to fetch + the file even though we don't own it. + """ + mirror_file = MIRROR_PATH + '/' + name + local_file = Path(get_distdir(), name) + cmd = [GSUTIL, 'cp', mirror_file, local_file] + logging.info('Running %r', cmd) + rc = subprocess.call(cmd) + if rc != 0: + logging.error( + """Could not fetch %s + +If the file does not yet exist at %s +please download the file, verify its integrity +with something like: + +curl -O https://static.rust-lang.org/dist/%s +gpg --verify %s.asc + +You may need to import the signing key first, e.g.: + +gpg --recv-keys 85AB96E6FA1BE5FE + +Once you have verify the integrity of the file, upload +it to the local mirror using gsutil cp. +""", mirror_file, MIRROR_PATH, name, name) + raise Exception(f'Could not fetch {mirror_file}') + # Check that the ACL allows allUsers READER access. + # If we get an AccessDeniedAcception here, that also + # counts as a success, because we were able to fetch + # the file as a non-owner. + cmd = [GSUTIL, 'acl', 'get', mirror_file] + logging.info('Running %r', cmd) + output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT) + acl_verified = False + if 'AccessDeniedException:' in output: + acl_verified = True + else: + acl = json.loads(output) + for x in acl: + if x['entity'] == 'allUsers' and x['role'] == 'READER': + acl_verified = True + break + if not acl_verified: + logging.error('Output from acl get:\n%s', output) + raise Exception('Could not verify that allUsers has READER permission') + + +def fetch_bootstrap_distfiles(old_version: RustVersion, + new_version: RustVersion) -> None: + """Fetches rust-bootstrap distfiles from the local mirror + + Fetches the distfiles for a rust-bootstrap ebuild to ensure they + are available on the mirror and the local copies are the same as + the ones on the mirror. + """ + fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version)) + fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) + + +def fetch_rust_distfiles(version: RustVersion) -> None: + """Fetches rust distfiles from the local mirror + + Fetches the distfiles for a rust ebuild to ensure they + are available on the mirror and the local copies are + the same as the ones on the mirror. + """ + fetch_distfile_from_mirror(compute_rustc_src_name(version)) + + +def get_distdir() -> os.PathLike: + """Returns portage's distdir.""" + return get_command_output(['portageq', 'distdir']) + + def update_manifest(ebuild_file: os.PathLike) -> None: + """Updates the MANIFEST for the ebuild at the given path.""" ebuild = Path(ebuild_file) logging.info('Added "mirror" to RESTRICT to %s', ebuild.name) flip_mirror_in_ebuild(ebuild, add=True) @@ -443,7 +558,7 @@ def prepare_uprev_from_json( def create_rust_uprev(rust_version: RustVersion, maybe_template_version: Optional[RustVersion], skip_compile: bool, run_step: Callable[[], T]) -> None: - template_version, template_ebuild, _old_bootstrap_version = run_step( + template_version, template_ebuild, old_bootstrap_version = run_step( 'prepare uprev', lambda: prepare_uprev(rust_version, maybe_template_version), result_from_json=prepare_uprev_from_json, @@ -451,6 +566,14 @@ def create_rust_uprev(rust_version: RustVersion, if template_ebuild is None: return + # The fetch steps will fail (on purpose) if the files they check for + # are not available on the mirror. To make them pass, fetch the + # required files yourself, verify their checksums, then upload them + # to the mirror. + run_step( + 'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles( + old_bootstrap_version, template_version)) + run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version)) run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild( template_version)) run_step( @@ -507,7 +630,7 @@ def remove_rust_bootstrap_version(version: RustVersion, prefix = f'rust-bootstrap-{version}' run_step('remove old bootstrap ebuild', lambda: remove_files( f'{prefix}*.ebuild', rust_bootstrap_path())) - ebuild_file = get_command_output(['equery', 'w', 'rust-bootstrap']) + ebuild_file = find_ebuild_for_package('rust-bootstrap') run_step('update bootstrap manifest to delete old version', lambda: update_manifest(ebuild_file)) @@ -532,7 +655,7 @@ def remove_rust_uprev(rust_version: Optional[RustVersion], 'remove patches', lambda: remove_files( f'files/rust-{delete_version}-*.patch', RUST_PATH)) run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH)) - ebuild_file = get_command_output(['equery', 'w', 'rust']) + ebuild_file = find_ebuild_for_package('rust') run_step('update manifest to delete old version', lambda: update_manifest( ebuild_file)) run_step('remove version from rust packages', lambda: update_rust_packages( @@ -561,7 +684,7 @@ def create_new_repo(rust_version: RustVersion) -> None: def build_cross_compiler() -> None: # Get target triples in ebuild - rust_ebuild = get_command_output(['equery', 'w', 'rust']) + rust_ebuild = find_ebuild_for_package('rust') with open(rust_ebuild, encoding='utf-8') as f: contents = f.read() diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py index 7b3c9e66..00761391 100755 --- a/rust_tools/rust_uprev_test.py +++ b/rust_tools/rust_uprev_test.py @@ -17,6 +17,54 @@ from unittest import mock from llvm_tools import git import rust_uprev +from rust_uprev import RustVersion + + +def _fail_command(cmd, *_args, **_kwargs): + err = subprocess.CalledProcessError(returncode=1, cmd=cmd) + err.stderr = b'mock failure' + raise err + + +class FetchDistfileTest(unittest.TestCase): + """Tests rust_uprev.fetch_distfile_from_mirror()""" + + @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') + @mock.patch.object(subprocess, 'call', side_effect=_fail_command) + def test_fetch_difstfile_fail(self, *_args) -> None: + with self.assertRaises(subprocess.CalledProcessError): + rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') + + @mock.patch.object(rust_uprev, + 'get_command_output_unchecked', + return_value='AccessDeniedException: Access denied.') + @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') + @mock.patch.object(subprocess, 'call', return_value=0) + def test_fetch_distfile_acl_access_denied(self, *_args) -> None: + rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') + + @mock.patch.object( + rust_uprev, + 'get_command_output_unchecked', + return_value='[ { "entity": "allUsers", "role": "READER" } ]') + @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') + @mock.patch.object(subprocess, 'call', return_value=0) + def test_fetch_distfile_acl_ok(self, *_args) -> None: + rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') + + @mock.patch.object( + rust_uprev, + 'get_command_output_unchecked', + return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]') + @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') + @mock.patch.object(subprocess, 'call', return_value=0) + def test_fetch_distfile_acl_wrong(self, *_args) -> None: + with self.assertRaisesRegex(Exception, 'allUsers.*READER'): + with self.assertLogs(level='ERROR') as log: + rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') + self.assertIn( + '[ { "entity": "___fake@google.com", "role": "OWNER" } ]', + '\n'.join(log.output)) class FindEbuildPathTest(unittest.TestCase): @@ -103,9 +151,12 @@ class PrepareUprevTest(unittest.TestCase): @mock.patch.object(rust_uprev, 'find_ebuild_for_rust_version', return_value='/path/to/ebuild') + @mock.patch.object(rust_uprev, + 'get_rust_bootstrap_version', + return_value=RustVersion(0, 41, 12)) @mock.patch.object(rust_uprev, 'get_command_output') def test_return_none_with_template_larger_than_input(self, mock_command, - _mock_find_ebuild): + *_args): ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=self.version_new) self.assertIsNone(ret) @@ -129,10 +180,13 @@ class PrepareUprevTest(unittest.TestCase): mock_command.assert_called_once_with(['equery', 'w', 'rust']) mock_exists.assert_not_called() + @mock.patch.object(rust_uprev, + 'get_rust_bootstrap_version', + return_value=RustVersion(0, 41, 12)) @mock.patch.object(os.path, 'exists') @mock.patch.object(rust_uprev, 'get_command_output') def test_return_none_with_ebuild_larger_than_input(self, mock_command, - mock_exists): + mock_exists, *_args): mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild' ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=None) @@ -350,8 +404,9 @@ class RustUprevOtherStagesTests(unittest.TestCase): ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=rust_uprev.RUST_PATH) + @mock.patch.object(rust_uprev, 'find_ebuild_for_package') @mock.patch.object(subprocess, 'check_call') - def test_remove_rust_bootstrap_version(self, mock_call): + def test_remove_rust_bootstrap_version(self, mock_call, *_args): bootstrap_path = os.path.join(rust_uprev.RUST_PATH, '..', 'rust-bootstrap') rust_uprev.remove_rust_bootstrap_version(self.old_version, lambda *x: ()) mock_call.has_calls([ diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index b9ad7b82..66df7c8a 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -9,8 +9,6 @@ Sends an email if something interesting (probably) happened. """ -# pylint: disable=cros-logging-import - import argparse import itertools import json diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index 97d111fc..a00f3ddc 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -6,8 +6,6 @@ """Tests for rust_watch.py.""" -# pylint: disable=cros-logging-import - import logging import pathlib import subprocess |