Merging 9 commit(s) from Chromium's toolchain-utils am: 3134544d54 am: e0c2630c64 am: 929241978e

Original change: https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/1845270

Change-Id: I497d318091cd0bb5742068f99bd7d3998962e1f0

14 files changed, 225 insertions, 48 deletions

diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json
index 3529ebb4..a4044c67 100644
--- a/afdo_metadata/kernel_afdo.json
+++ b/afdo_metadata/kernel_afdo.json
@@ -1,17 +1,14 @@
 {
-    "chromeos-kernel-3_18": {
-        "name": "R94-14092.5-1627896981"
-    },
     "chromeos-kernel-4_4": {
-        "name": "R94-14119.0-1627896842"
+        "name": "R96-14209.0-1632130382"
     },
     "chromeos-kernel-4_14": {
-        "name": "R94-14119.0-1627896837"
+        "name": "R96-14221.0-1632130642"
     },
     "chromeos-kernel-4_19": {
-        "name": "R94-14119.0-1627896728"
+        "name": "R96-14209.0-1632130511"
     },
     "chromeos-kernel-5_4": {
-        "name": "R94-14119.0-1627896791"
+        "name": "R96-14209.0-1632130436"
     }
 }
diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo
index 4b41053f..ff0ab224 100755
--- a/afdo_tools/update_kernel_afdo
+++ b/afdo_tools/update_kernel_afdo
@@ -25,8 +25,10 @@ set -eu
 set -o pipefail
 
 GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel
-KVERS="3.18 4.4 4.14 4.19 5.4"
+KVERS="4.4 4.14 4.19 5.4"
 failed_channels=""
+# Add skipped chrome branches in ascending order here.
+SKIPPED_BRANCHES="95"
 
 script_dir=$(dirname "$0")
 tc_utils_dir="${script_dir}/.."
@@ -57,8 +59,14 @@ branch["canary"]=${canary_ref##*/}
 # Get current branch numbers (number which goes after R).
 branch_number["stable"]=$(echo "${branch["stable"]}" | \
   sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p")
-branch_number["beta"]="$((branch_number[stable] + 1))"
+branch_number["beta"]=$(echo "${branch["beta"]}" | \
+  sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p")
 branch_number["canary"]="$((branch_number[beta] + 1))"
+for skipped_branch in $SKIPPED_BRANCHES ; do
+  if [[ ${branch_number["canary"]} == $skipped_branch ]] ; then
+    ((branch_number[canary]++))
+  fi
+done
 
 # Without arguments the script updates all branches.
 channels=${1:-"all"}
@@ -180,7 +188,6 @@ EOT
     canary )
       commit_contents="afdo_metadata: Publish the new kernel profiles
 
-Update chromeos-kernel-3_18
 Update chromeos-kernel-4_4
 Update chromeos-kernel-4_14
 Update chromeos-kernel-4_19
diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go
index 7d949d34..986eabab 100644
--- a/compiler_wrapper/compiler_wrapper.go
+++ b/compiler_wrapper/compiler_wrapper.go
@@ -74,25 +74,36 @@ func runAndroidClangTidy(env env, cmd *command) error {
 	if err != nil || seconds == 0 {
 		return env.exec(cmd)
 	}
+	getSourceFile := func() string {
+		// Note: This depends on Android build system's clang-tidy command line format.
+		// Last non-flag before "--" in cmd.Args is used as the source file name.
+		sourceFile := "unknown_file"
+		for _, arg := range cmd.Args {
+			if arg == "--" {
+				break
+			}
+			if strings.HasPrefix(arg, "-") {
+				continue
+			}
+			sourceFile = arg
+		}
+		return sourceFile
+	}
+	startTime := time.Now()
 	err = env.runWithTimeout(cmd, time.Duration(seconds)*time.Second)
 	if !errors.Is(err, context.DeadlineExceeded) {
+		// When used time is over half of TIDY_TIMEOUT, give a warning.
+		// These warnings allow users to fix slow jobs before they get worse.
+		usedSeconds := int(time.Now().Sub(startTime) / time.Second)
+		if usedSeconds > seconds/2 {
+			warning := "%s:1:1: warning: clang-tidy used %d seconds.\n"
+			fmt.Fprintf(env.stdout(), warning, getSourceFile(), usedSeconds)
+		}
 		return err
 	}
 	// When DeadllineExceeded, print warning messages.
-	// Note: This depends on Android build system's clang-tidy command line format.
-	// Last non-flag before "--" in cmd.Args is used as the source file name.
-	sourceFile := "unknown_file"
-	for _, arg := range cmd.Args {
-		if arg == "--" {
-			break
-		}
-		if strings.HasPrefix(arg, "-") {
-			continue
-		}
-		sourceFile = arg
-	}
 	warning := "%s:1:1: warning: clang-tidy aborted after %d seconds.\n"
-	fmt.Fprintf(env.stdout(), warning, sourceFile, seconds)
+	fmt.Fprintf(env.stdout(), warning, getSourceFile(), seconds)
 	fmt.Fprintf(env.stdout(), "TIMEOUT: %s %s\n", cmd.Path, strings.Join(cmd.Args, " "))
 	// Do not stop Android build. Just give a warning and return no error.
 	return nil
diff --git a/cwp/cr-os/fetch_gn_descs.py b/cwp/cr-os/fetch_gn_descs.py
index 60d331c5..8a0b2e4e 100755
--- a/cwp/cr-os/fetch_gn_descs.py
+++ b/cwp/cr-os/fetch_gn_descs.py
@@ -23,7 +23,6 @@ from __future__ import print_function
 
 import argparse
 import json
-# pylint: disable=cros-logging-import
 import logging
 import os
 import subprocess
diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py
index e8ee2ab6..c53db179 100755
--- a/llvm_tools/bisect_clang_crashes.py
+++ b/llvm_tools/bisect_clang_crashes.py
@@ -7,8 +7,6 @@
 """Fetches and submits the artifacts from Chrome OS toolchain's crash bucket.
 """
 
-# pylint: disable=cros-logging-import
-
 import argparse
 import glob
 import json
diff --git a/llvm_tools/bisect_clang_crashes_unittest.py b/llvm_tools/bisect_clang_crashes_unittest.py
index c9143450..a3dc0c6d 100755
--- a/llvm_tools/bisect_clang_crashes_unittest.py
+++ b/llvm_tools/bisect_clang_crashes_unittest.py
@@ -6,7 +6,6 @@
 
 """Tests for bisect_clang_crashes."""
 
-# pylint: disable=cros-logging-import
 import glob
 import logging
 import os.path
diff --git a/llvm_tools/fetch_cros_sdk_rolls.py b/llvm_tools/fetch_cros_sdk_rolls.py
index 42af678a..83d7025a 100755
--- a/llvm_tools/fetch_cros_sdk_rolls.py
+++ b/llvm_tools/fetch_cros_sdk_rolls.py
@@ -10,8 +10,6 @@ gs://chromiumos-sdk/. The hope is that it'll help answer the question "when did
 the toolchain ebuild ${x} go live?"
 """
 
-# pylint: disable=cros-logging-import
-
 import argparse
 import json
 import logging
diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py
index 1a7bc793..6941f3d6 100755
--- a/llvm_tools/nightly_revert_checker.py
+++ b/llvm_tools/nightly_revert_checker.py
@@ -10,8 +10,6 @@ If any reverts are found that were previously unknown, this cherry-picks them or
 fires off an email. All LLVM SHAs to monitor are autodetected.
 """
 
-# pylint: disable=cros-logging-import
-
 from __future__ import print_function
 
 import argparse
diff --git a/llvm_tools/upload_lexan_crashes_to_forcey.py b/llvm_tools/upload_lexan_crashes_to_forcey.py
index 486fe664..61bf6b7d 100755
--- a/llvm_tools/upload_lexan_crashes_to_forcey.py
+++ b/llvm_tools/upload_lexan_crashes_to_forcey.py
@@ -6,8 +6,6 @@
 
 """Fetches and submits the latest test-cases from Lexan's crash bucket."""
 
-# pylint: disable=cros-logging-import
-
 import argparse
 import contextlib
 import datetime
diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py
index cb41eb88..be159b94 100755
--- a/pgo_tools/monitor_pgo_profiles.py
+++ b/pgo_tools/monitor_pgo_profiles.py
@@ -5,8 +5,6 @@
 
 """Emails the mage if PGO profile generation hasn't succeeded recently."""
 
-# pylint: disable=cros-logging-import
-
 import argparse
 import datetime
 import sys
diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
index 16fb0da8..011639df 100755
--- a/rust_tools/rust_uprev.py
+++ b/rust_tools/rust_uprev.py
@@ -46,6 +46,10 @@ from pathlib import Path
 from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple
 
 from llvm_tools import chroot, git
+
+EQUERY = 'equery'
+GSUTIL = 'gsutil.py'
+MIRROR_PATH = 'gs://chromeos-localmirror/distfiles'
 RUST_PATH = Path(
     '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust')
 
@@ -55,6 +59,15 @@ def get_command_output(command: List[str], *args, **kwargs) -> str:
                                  **kwargs).strip()
 
 
+def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str:
+  return subprocess.run(command,
+                        check=False,
+                        stdout=subprocess.PIPE,
+                        encoding='utf-8',
+                        *args,
+                        **kwargs).stdout.strip()
+
+
 class RustVersion(NamedTuple):
   """NamedTuple represents a Rust version"""
   major: int
@@ -90,6 +103,19 @@ class RustVersion(NamedTuple):
                        int(m.group('patch')))
 
 
+def compute_rustc_src_name(version: RustVersion) -> str:
+  return f'rustc-{version}-src.tar.gz'
+
+
+def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str:
+  return f'rust-bootstrap-{version}.tbz2'
+
+
+def find_ebuild_for_package(name: str) -> os.PathLike:
+  """Returns the path to the ebuild for the named package."""
+  return get_command_output([EQUERY, 'w', name])
+
+
 def find_ebuild_path(directory: Path,
                      name: str,
                      version: Optional[RustVersion] = None) -> Path:
@@ -252,7 +278,7 @@ def parse_commandline_args() -> argparse.Namespace:
 def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion]
                   ) -> Optional[Tuple[RustVersion, str, RustVersion]]:
   if template is None:
-    ebuild_path = get_command_output(['equery', 'w', 'rust'])
+    ebuild_path = find_ebuild_for_package('rust')
     ebuild_name = os.path.basename(ebuild_path)
     template_version = RustVersion.parse_from_ebuild(ebuild_name)
   else:
@@ -354,14 +380,103 @@ def flip_mirror_in_ebuild(ebuild_file: Path, add: bool) -> None:
 
 def ebuild_actions(package: str, actions: List[str],
                    sudo: bool = False) -> None:
-  ebuild_path_inchroot = get_command_output(['equery', 'w', package])
+  ebuild_path_inchroot = find_ebuild_for_package(package)
   cmd = ['ebuild', ebuild_path_inchroot] + actions
   if sudo:
     cmd = ['sudo'] + cmd
   subprocess.check_call(cmd)
 
 
+def fetch_distfile_from_mirror(name: str) -> None:
+  """Gets the named file from the local mirror.
+
+  This ensures that the file exists on the mirror, and
+  that we can read it. We overwrite any existing distfile
+  to ensure the checksums that update_manifest() records
+  match the file as it exists on the mirror.
+
+  This function also attempts to verify the ACL for
+  the file (which is expected to have READER permission
+  for allUsers). We can only see the ACL if the user
+  gsutil runs with is the owner of the file. If not,
+  we get an access denied error. We also count this
+  as a success, because it means we were able to fetch
+  the file even though we don't own it.
+  """
+  mirror_file = MIRROR_PATH + '/' + name
+  local_file = Path(get_distdir(), name)
+  cmd = [GSUTIL, 'cp', mirror_file, local_file]
+  logging.info('Running %r', cmd)
+  rc = subprocess.call(cmd)
+  if rc != 0:
+    logging.error(
+        """Could not fetch %s
+
+If the file does not yet exist at %s
+please download the file, verify its integrity
+with something like:
+
+curl -O https://static.rust-lang.org/dist/%s
+gpg --verify %s.asc
+
+You may need to import the signing key first, e.g.:
+
+gpg --recv-keys 85AB96E6FA1BE5FE
+
+Once you have verify the integrity of the file, upload
+it to the local mirror using gsutil cp.
+""", mirror_file, MIRROR_PATH, name, name)
+    raise Exception(f'Could not fetch {mirror_file}')
+  # Check that the ACL allows allUsers READER access.
+  # If we get an AccessDeniedAcception here, that also
+  # counts as a success, because we were able to fetch
+  # the file as a non-owner.
+  cmd = [GSUTIL, 'acl', 'get', mirror_file]
+  logging.info('Running %r', cmd)
+  output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT)
+  acl_verified = False
+  if 'AccessDeniedException:' in output:
+    acl_verified = True
+  else:
+    acl = json.loads(output)
+    for x in acl:
+      if x['entity'] == 'allUsers' and x['role'] == 'READER':
+        acl_verified = True
+        break
+  if not acl_verified:
+    logging.error('Output from acl get:\n%s', output)
+    raise Exception('Could not verify that allUsers has READER permission')
+
+
+def fetch_bootstrap_distfiles(old_version: RustVersion,
+                              new_version: RustVersion) -> None:
+  """Fetches rust-bootstrap distfiles from the local mirror
+
+  Fetches the distfiles for a rust-bootstrap ebuild to ensure they
+  are available on the mirror and the local copies are the same as
+  the ones on the mirror.
+  """
+  fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version))
+  fetch_distfile_from_mirror(compute_rustc_src_name(new_version))
+
+
+def fetch_rust_distfiles(version: RustVersion) -> None:
+  """Fetches rust distfiles from the local mirror
+
+  Fetches the distfiles for a rust ebuild to ensure they
+  are available on the mirror and the local copies are
+  the same as the ones on the mirror.
+  """
+  fetch_distfile_from_mirror(compute_rustc_src_name(version))
+
+
+def get_distdir() -> os.PathLike:
+  """Returns portage's distdir."""
+  return get_command_output(['portageq', 'distdir'])
+
+
 def update_manifest(ebuild_file: os.PathLike) -> None:
+  """Updates the MANIFEST for the ebuild at the given path."""
   ebuild = Path(ebuild_file)
   logging.info('Added "mirror" to RESTRICT to %s', ebuild.name)
   flip_mirror_in_ebuild(ebuild, add=True)
@@ -443,7 +558,7 @@ def prepare_uprev_from_json(
 def create_rust_uprev(rust_version: RustVersion,
                       maybe_template_version: Optional[RustVersion],
                       skip_compile: bool, run_step: Callable[[], T]) -> None:
-  template_version, template_ebuild, _old_bootstrap_version = run_step(
+  template_version, template_ebuild, old_bootstrap_version = run_step(
       'prepare uprev',
       lambda: prepare_uprev(rust_version, maybe_template_version),
       result_from_json=prepare_uprev_from_json,
@@ -451,6 +566,14 @@ def create_rust_uprev(rust_version: RustVersion,
   if template_ebuild is None:
     return
 
+  # The fetch steps will fail (on purpose) if the files they check for
+  # are not available on the mirror. To make them pass, fetch the
+  # required files yourself, verify their checksums, then upload them
+  # to the mirror.
+  run_step(
+      'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles(
+          old_bootstrap_version, template_version))
+  run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version))
   run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild(
       template_version))
   run_step(
@@ -507,7 +630,7 @@ def remove_rust_bootstrap_version(version: RustVersion,
   prefix = f'rust-bootstrap-{version}'
   run_step('remove old bootstrap ebuild', lambda: remove_files(
       f'{prefix}*.ebuild', rust_bootstrap_path()))
-  ebuild_file = get_command_output(['equery', 'w', 'rust-bootstrap'])
+  ebuild_file = find_ebuild_for_package('rust-bootstrap')
   run_step('update bootstrap manifest to delete old version', lambda:
            update_manifest(ebuild_file))
 
@@ -532,7 +655,7 @@ def remove_rust_uprev(rust_version: Optional[RustVersion],
       'remove patches', lambda: remove_files(
           f'files/rust-{delete_version}-*.patch', RUST_PATH))
   run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH))
-  ebuild_file = get_command_output(['equery', 'w', 'rust'])
+  ebuild_file = find_ebuild_for_package('rust')
   run_step('update manifest to delete old version', lambda: update_manifest(
       ebuild_file))
   run_step('remove version from rust packages', lambda: update_rust_packages(
@@ -561,7 +684,7 @@ def create_new_repo(rust_version: RustVersion) -> None:
 
 def build_cross_compiler() -> None:
   # Get target triples in ebuild
-  rust_ebuild = get_command_output(['equery', 'w', 'rust'])
+  rust_ebuild = find_ebuild_for_package('rust')
   with open(rust_ebuild, encoding='utf-8') as f:
     contents = f.read()
 
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
index 7b3c9e66..00761391 100755
--- a/rust_tools/rust_uprev_test.py
+++ b/rust_tools/rust_uprev_test.py
@@ -17,6 +17,54 @@ from unittest import mock
 from llvm_tools import git
 
 import rust_uprev
+from rust_uprev import RustVersion
+
+
+def _fail_command(cmd, *_args, **_kwargs):
+  err = subprocess.CalledProcessError(returncode=1, cmd=cmd)
+  err.stderr = b'mock failure'
+  raise err
+
+
+class FetchDistfileTest(unittest.TestCase):
+  """Tests rust_uprev.fetch_distfile_from_mirror()"""
+
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', side_effect=_fail_command)
+  def test_fetch_difstfile_fail(self, *_args) -> None:
+    with self.assertRaises(subprocess.CalledProcessError):
+      rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+  @mock.patch.object(rust_uprev,
+                     'get_command_output_unchecked',
+                     return_value='AccessDeniedException: Access denied.')
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', return_value=0)
+  def test_fetch_distfile_acl_access_denied(self, *_args) -> None:
+    rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+  @mock.patch.object(
+      rust_uprev,
+      'get_command_output_unchecked',
+      return_value='[ { "entity": "allUsers", "role": "READER" } ]')
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', return_value=0)
+  def test_fetch_distfile_acl_ok(self, *_args) -> None:
+    rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+  @mock.patch.object(
+      rust_uprev,
+      'get_command_output_unchecked',
+      return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]')
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', return_value=0)
+  def test_fetch_distfile_acl_wrong(self, *_args) -> None:
+    with self.assertRaisesRegex(Exception, 'allUsers.*READER'):
+      with self.assertLogs(level='ERROR') as log:
+        rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+        self.assertIn(
+            '[ { "entity": "___fake@google.com", "role": "OWNER" } ]',
+            '\n'.join(log.output))
 
 
 class FindEbuildPathTest(unittest.TestCase):
@@ -103,9 +151,12 @@ class PrepareUprevTest(unittest.TestCase):
   @mock.patch.object(rust_uprev,
                      'find_ebuild_for_rust_version',
                      return_value='/path/to/ebuild')
+  @mock.patch.object(rust_uprev,
+                     'get_rust_bootstrap_version',
+                     return_value=RustVersion(0, 41, 12))
   @mock.patch.object(rust_uprev, 'get_command_output')
   def test_return_none_with_template_larger_than_input(self, mock_command,
-                                                       _mock_find_ebuild):
+                                                       *_args):
     ret = rust_uprev.prepare_uprev(rust_version=self.version_old,
                                    template=self.version_new)
     self.assertIsNone(ret)
@@ -129,10 +180,13 @@ class PrepareUprevTest(unittest.TestCase):
     mock_command.assert_called_once_with(['equery', 'w', 'rust'])
     mock_exists.assert_not_called()
 
+  @mock.patch.object(rust_uprev,
+                     'get_rust_bootstrap_version',
+                     return_value=RustVersion(0, 41, 12))
   @mock.patch.object(os.path, 'exists')
   @mock.patch.object(rust_uprev, 'get_command_output')
   def test_return_none_with_ebuild_larger_than_input(self, mock_command,
-                                                     mock_exists):
+                                                     mock_exists, *_args):
     mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild'
     ret = rust_uprev.prepare_uprev(rust_version=self.version_old,
                                    template=None)
@@ -350,8 +404,9 @@ class RustUprevOtherStagesTests(unittest.TestCase):
         ['git', 'add', f'rust-{self.new_version}.ebuild'],
         cwd=rust_uprev.RUST_PATH)
 
+  @mock.patch.object(rust_uprev, 'find_ebuild_for_package')
   @mock.patch.object(subprocess, 'check_call')
-  def test_remove_rust_bootstrap_version(self, mock_call):
+  def test_remove_rust_bootstrap_version(self, mock_call, *_args):
     bootstrap_path = os.path.join(rust_uprev.RUST_PATH, '..', 'rust-bootstrap')
     rust_uprev.remove_rust_bootstrap_version(self.old_version, lambda *x: ())
     mock_call.has_calls([
diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py
index b9ad7b82..66df7c8a 100755
--- a/rust_tools/rust_watch.py
+++ b/rust_tools/rust_watch.py
@@ -9,8 +9,6 @@
 Sends an email if something interesting (probably) happened.
 """
 
-# pylint: disable=cros-logging-import
-
 import argparse
 import itertools
 import json
diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py
index 97d111fc..a00f3ddc 100755
--- a/rust_tools/rust_watch_test.py
+++ b/rust_tools/rust_watch_test.py
@@ -6,8 +6,6 @@
 
 """Tests for rust_watch.py."""
 
-# pylint: disable=cros-logging-import
-
 import logging
 import pathlib
 import subprocess