aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPirama Arumuga Nainar <pirama@google.com>2020-10-22 08:24:15 +0000
committerAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>2020-10-22 08:24:15 +0000
commit7a702fb88be5cf729d1a248d049e50ca1d35ed16 (patch)
tree073e9e403bc6e2290b2b5b005a5cfdabadde1e42
parentbdd29acf271dfaee411a6e3434c0d2bd612834a1 (diff)
parent850d25c38cd14586fd263215480c3176fbd36480 (diff)
downloadtoolchain-utils-7a702fb88be5cf729d1a248d049e50ca1d35ed16.tar.gz
Merging 84 commit(s) from Chromium's toolchain-utils am: 1e576757bc am: 12fcdf982e am: 76b8300539 am: 850d25c38c
Original change: https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/1469797 Change-Id: I0073071a09fd8710738e053a8726e21bfba0b1c8
-rw-r--r--OWNERS.toolchain2
-rw-r--r--afdo_metadata/kernel_afdo.json12
-rwxr-xr-xafdo_tools/update_kernel_afdo118
-rwxr-xr-xandroid_bench_suite/fix_skia_results.py21
-rwxr-xr-xandroid_merge_from_upstream.sh22
-rwxr-xr-xauto_delete_nightly_test_data.py37
-rwxr-xr-xbinary_search_tool/common/test_setup.sh16
-rwxr-xr-xbuildbot_test_toolchains.py2
-rw-r--r--compiler_wrapper/README.md53
-rw-r--r--compiler_wrapper/bisect_flag.go3
-rw-r--r--compiler_wrapper/clang_flags.go10
-rw-r--r--compiler_wrapper/clang_flags_test.go8
-rw-r--r--compiler_wrapper/clang_tidy_flag.go215
-rw-r--r--compiler_wrapper/clang_tidy_flag_test.go170
-rw-r--r--compiler_wrapper/compiler_wrapper.go73
-rw-r--r--compiler_wrapper/config.go44
-rw-r--r--compiler_wrapper/cros_llvm_next_flags.go9
-rwxr-xr-xcompiler_wrapper/install_compiler_wrapper.sh (renamed from compiler_wrapper/update_compiler_wrapper.sh)0
-rw-r--r--compiler_wrapper/libc_exec.go22
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/bisect.json9
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json27
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json3
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json27
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json36
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json24
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json29
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json24
-rw-r--r--compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json15
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/bisect.json12
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json30
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json30
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clang_path.json48
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json32
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json36
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json23
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json32
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json20
-rw-r--r--compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json16
-rw-r--r--compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json21
-rw-r--r--compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json84
-rw-r--r--compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json56
-rw-r--r--compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json35
-rw-r--r--compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json28
-rw-r--r--compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json12
-rw-r--r--compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json48
-rw-r--r--compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json32
-rw-r--r--compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json20
-rw-r--r--compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json16
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json9
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json21
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json21
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json36
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json24
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json29
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json17
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json24
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json15
-rw-r--r--compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json12
-rw-r--r--compiler_wrapper/testutil_test.go2
-rwxr-xr-xcros_utils/buildbot_json.py1534
-rw-r--r--cros_utils/buildbot_utils.py20
-rwxr-xr-xcros_utils/buildbot_utils_unittest.py12
-rwxr-xr-xcros_utils/command_executer.py60
-rw-r--r--cros_utils/manifest_versions.py12
-rw-r--r--cros_utils/misc.py10
-rw-r--r--cros_utils/tabulator.py90
-rwxr-xr-xcros_utils/tabulator_test.py14
-rw-r--r--crosperf/default_remotes17
-rw-r--r--crosperf/experiment_runner.py36
-rwxr-xr-xcrosperf/generate_report_unittest.py6
-rw-r--r--crosperf/label.py36
-rw-r--r--crosperf/results_cache.py134
-rwxr-xr-xcrosperf/results_cache_unittest.py194
-rw-r--r--crosperf/results_report.py11
-rwxr-xr-xcrosperf/results_report_unittest.py3
-rw-r--r--debug_info_test/allowlist.py (renamed from debug_info_test/whitelist.py)28
-rw-r--r--debug_info_test/check_exist.py10
-rw-r--r--debug_info_test/check_ngcc.py6
-rw-r--r--debug_info_test/exist_debug_info.allowlist (renamed from debug_info_test/exist_debug_info.whitelist)0
-rw-r--r--debug_info_test/exist_producer.allowlist (renamed from debug_info_test/exist_producer.whitelist)0
-rw-r--r--debug_info_test/ngcc_comp_path.allowlist (renamed from debug_info_test/ngcc_comp_path.whitelist)0
-rw-r--r--debug_info_test/ngcc_dso_path.allowlist (renamed from debug_info_test/ngcc_dso_path.whitelist)2
-rw-r--r--llvm_tools/README.md44
-rwxr-xr-xllvm_tools/bisect_clang_crashes.py139
-rwxr-xr-xllvm_tools/bisect_clang_crashes_unittest.py92
-rwxr-xr-xllvm_tools/cherrypick_cl.py34
-rwxr-xr-xllvm_tools/fetch_cros_sdk_rolls.py112
-rwxr-xr-xllvm_tools/get_llvm_hash.py46
-rwxr-xr-xllvm_tools/get_llvm_hash_unittest.py5
-rwxr-xr-xllvm_tools/git_llvm_rev.py14
-rwxr-xr-xllvm_tools/git_llvm_rev_test.py30
-rwxr-xr-xllvm_tools/llvm_bisection.py278
-rwxr-xr-xllvm_tools/llvm_bisection_unittest.py624
-rw-r--r--llvm_tools/llvm_project.py11
-rwxr-xr-xllvm_tools/nightly_revert_checker.py9
-rwxr-xr-xllvm_tools/upload_lexan_crashes_to_forcey.py258
-rwxr-xr-xllvm_tools/upload_lexan_crashes_to_forcey_test.py122
-rwxr-xr-xpgo_tools/monitor_pgo_profiles.py150
-rwxr-xr-xpgo_tools/monitor_pgo_profiles_unittest.py100
-rwxr-xr-xrust_tools/rust_uprev.py673
-rwxr-xr-xrust_tools/rust_uprev_test.py455
-rwxr-xr-xrust_tools/rust_watch.py351
-rwxr-xr-xrust_tools/rust_watch_test.py202
-rwxr-xr-xsetup_chromeos.py2
-rwxr-xr-xtoolchain_utils_githooks/check-presubmit.py96
-rw-r--r--unblocked_terms.txt5
106 files changed, 5240 insertions, 2719 deletions
diff --git a/OWNERS.toolchain b/OWNERS.toolchain
index f7d6e3f6..48ecc260 100644
--- a/OWNERS.toolchain
+++ b/OWNERS.toolchain
@@ -1,10 +1,10 @@
cmtice@chromium.org
denik@chromium.org
gbiv@chromium.org
+inglorion@chromium.org
jiancai@chromium.org
llozano@chromium.org
manojgupta@chromium.org
-tbosch@chromium.org
tcwang@chromium.org
zhizhouy@chromium.org
diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json
index a6f289f2..2f4ead4a 100644
--- a/afdo_metadata/kernel_afdo.json
+++ b/afdo_metadata/kernel_afdo.json
@@ -1,14 +1,14 @@
{
- "chromeos-kernel-4_4": {
- "name": "R85-13280.0-1592213800"
- },
"chromeos-kernel-3_18": {
- "name": "R85-13280.0-1592213569"
+ "name": "R88-13502.0-1602497202"
+ },
+ "chromeos-kernel-4_4": {
+ "name": "R88-13505.4-1602496005"
},
"chromeos-kernel-4_14": {
- "name": "R85-13280.0-1592214019"
+ "name": "R88-13505.4-1602496823"
},
"chromeos-kernel-4_19": {
- "name": "R85-13280.0-1592214191"
+ "name": "R88-13502.0-1602496205"
}
}
diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo
new file mode 100755
index 00000000..8eb1306c
--- /dev/null
+++ b/afdo_tools/update_kernel_afdo
@@ -0,0 +1,118 @@
+#!/bin/bash
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Due to crbug.com/1081332, we need to update AFDO metadata
+# manually. This script performs a few checks and generates a
+# new kernel_afdo.json file, which can then be committed.
+#
+# USAGE:
+# toolchain-utils$ ./afdo_tools/update_kernel_afdo
+#
+# The script modifies the JSON file and shows the git diff.
+#
+# If the changes look good, git commit them. Example commit
+# message (from crrev.com/c/2197462):
+#
+# afdo_metadata: Publish the new kernel profiles
+#
+# Update chromeos-kernel-3_18 to R84-13080.0-1589189810
+# Update chromeos-kernel-4_4 to R84-13080.0-1589189726
+# Update chromeos-kernel-4_14 to R84-13080.0-1589190025
+# Update chromeos-kernel-4_19 to R84-13080.0-1589189550
+#
+# BUG=None
+# TEST=Verified in kernel-release-afdo-verify-orchestrator.
+#
+
+set -eu
+set -o pipefail
+
+CROS_REPO=https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay
+GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel
+KVERS="3.18 4.4 4.14 4.19"
+errs=""
+successes=0
+
+script_dir=$(dirname "$0")
+tc_utils_dir="$script_dir/.."
+metadata_dir="$tc_utils_dir/afdo_metadata"
+outfile="$metadata_dir/kernel_afdo.json"
+
+# The most recent Monday, in Unix timestamp format.
+if [ $(date +%a) = "Mon" ]
+then
+ expected_time=$(date +%s -d 00:00:00)
+else
+ expected_time=$(date +%s -d "last Monday")
+fi
+
+# Get the current canary branch number (using beta + 1)
+beta=$(git ls-remote -h $CROS_REPO | \
+ sed -n -e "s/^.*release-R\([0-9][0-9]*\).*$/\1/p" | \
+ sort -g | tail -1)
+canary="$(($beta + 1))"
+
+json="{"
+sep=""
+for kver in $KVERS
+do
+ # Sort the gs output by timestamp (default ordering is by name, so
+ # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz)
+ latest=$(gsutil ls -l "$GS_BASE/$kver/" | sort -k2 | \
+ grep "R${canary}" | tail -1 || true)
+ if [ -z "$latest" ]
+ then
+ # if no profiles exist for R${canary}, try the previous branch
+ latest=$(gsutil ls -l "$GS_BASE/$kver/" | sort -k2 | \
+ grep "R${beta}" | tail -1)
+ fi
+
+ # Verify that the file has the expected date.
+ file_time=$(echo "$latest" | awk '{print $2}')
+ file_time_unix=$(date +%s -d "$file_time")
+ if [ $file_time_unix -lt $expected_time ]
+ then
+ expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time)
+ echo "Wrong date for $kver: $file_time is before $expected" >&2
+ errs="$errs $kver"
+ continue
+ fi
+
+ # Generate JSON.
+ json_kver=$(echo "$kver" | tr . _)
+ # b/147370213 (migrating profiles from gcov format) may result in the
+ # pattern below no longer doing the right thing.
+ name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%')
+ json=$(cat <<EOT
+$json$sep
+ "chromeos-kernel-$json_kver": {
+ "name": "$name"
+ }
+EOT
+)
+ sep=","
+ successes=$((successes + 1))
+done
+
+# If we did not succeed for any kvers, exit now.
+if [ $successes -eq 0 ]
+then
+ echo "error: AFDO profiles out of date for all kernel versions" >&2
+ exit 2
+fi
+
+# Write new JSON file.
+printf "%s\n}\n" "$json" > "$outfile"
+
+# Show the changes.
+(cd "$tc_utils_dir" && git diff)
+
+# If no changes were made, say so.
+outdir=$(dirname "$outfile")
+shortstat=$(cd "$outdir" && git status --short $(basename "$outfile"))
+[ -n "$shortstat" ] || echo $(basename "$outfile")" is up to date."
+
+# If we had any errors, warn about them.
+[ -z "$errs" ] || echo "warning: failed to update$errs" >&2
diff --git a/android_bench_suite/fix_skia_results.py b/android_bench_suite/fix_skia_results.py
index 8c919d35..84dee5a5 100755
--- a/android_bench_suite/fix_skia_results.py
+++ b/android_bench_suite/fix_skia_results.py
@@ -5,6 +5,7 @@
# found in the LICENSE file.
#
# pylint: disable=cros-logging-import
+
"""Transforms skia benchmark results to ones that crosperf can understand."""
from __future__ import print_function
@@ -57,7 +58,10 @@ def _GetTimeMultiplier(label_name):
def _GetTimeDenom(ms):
- """Given a list of times (in milliseconds), find a sane time unit for them.
+ """Express times in a common time unit.
+
+ Given a list of times (in milliseconds), find a time unit in which
+ they can all be expressed.
Returns the unit name, and `ms` normalized to that time unit.
@@ -94,9 +98,9 @@ def _TransformBenchmarks(raw_benchmarks):
# statistic...
benchmarks = raw_benchmarks['results']
results = []
- for bench_name, bench_result in benchmarks.iteritems():
+ for bench_name, bench_result in benchmarks.items():
try:
- for cfg_name, keyvals in bench_result.iteritems():
+ for cfg_name, keyvals in bench_result.items():
# Some benchmarks won't have timing data (either it won't exist at all,
# or it'll be empty); skip them.
samples = keyvals.get('samples')
@@ -109,17 +113,16 @@ def _TransformBenchmarks(raw_benchmarks):
friendly_name = _GetFamiliarName(bench_name)
if len(results) < len(samples):
- results.extend({
- 'retval': 0
- } for _ in range(len(samples) - len(results)))
+ results.extend(
+ {'retval': 0} for _ in range(len(samples) - len(results)))
time_mul = _GetTimeMultiplier(friendly_name)
- for sample, app in itertools.izip(samples, results):
+ for sample, app in itertools.zip(samples, results):
assert friendly_name not in app
app[friendly_name] = sample * time_mul
except (KeyError, ValueError) as e:
- logging.error('While converting "%s" (key: %s): %s',
- bench_result, bench_name, e.message)
+ logging.error('While converting "%s" (key: %s): %s', bench_result,
+ bench_name, e)
raise
# Realistically, [results] should be multiple results, where each entry in the
diff --git a/android_merge_from_upstream.sh b/android_merge_from_upstream.sh
index cf07d4bf..b63eb6e1 100755
--- a/android_merge_from_upstream.sh
+++ b/android_merge_from_upstream.sh
@@ -18,11 +18,11 @@ remote_branch="${remote}/upstream-mirror-master"
my_dir="$(dirname "$(readlink -m "$0")")"
cd "${my_dir}"
-ensure_head_is_upstream_master() {
- local current_rev master_rev
+ensure_head_is_upstream_main() {
+ local current_rev main_rev
current_rev="$(git rev-parse HEAD)"
- master_rev="$(git rev-parse ${local_upstream})"
- if [[ "${current_rev}" != "${master_rev}" ]]; then
+ main_rev="$(git rev-parse ${local_upstream})"
+ if [[ "${current_rev}" != "${main_rev}" ]]; then
echo "Please checkout ${local_upstream} and rerun this" >&2
exit
fi
@@ -42,16 +42,8 @@ ensure_no_local_branch_present() {
exit 1
fi
- # If we're *on* that branch, deleting it is difficult.
- local current_branch
- current_branch="$(git branch --show-current)"
- if [[ "${current_branch}" == "${local_branch_name}" ]]; then
- local rev
- rev="$(git rev-parse HEAD)"
- # This is fine, since we assume HEAD == upstream-mirror-master anyway
- # (e.g., the existing branch was pointless.)
- git checkout "${rev}"
- fi
+ # If we're *on* that branch, deleting it is difficult. Always detach.
+ git checkout --detach || return
git branch -D "${local_branch_name}"
}
@@ -61,7 +53,7 @@ get_merge_commit_list() {
git log --oneline "${merge_base}..${remote_branch}"
}
-ensure_head_is_upstream_master
+ensure_head_is_upstream_main
ensure_no_local_branch_present
echo "Ensuring repository is up-to-date..."
diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py
index e40b95c2..c3c2e24c 100755
--- a/auto_delete_nightly_test_data.py
+++ b/auto_delete_nightly_test_data.py
@@ -16,6 +16,7 @@ import datetime
import os
import re
import shutil
+import shlex
import sys
import time
@@ -49,10 +50,10 @@ def CleanNumberedDir(s, dry_run=False):
return False
## Now delete the numbered dir Before forcibly removing the directory, just
- ## check 's' to make sure it is sane. A valid dir to be removed must be
- ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'.
- valid_dir_pattern = (
- '^' + NIGHTLY_TESTS_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')')
+ ## check 's' to make sure it matches the expected pattern. A valid dir to be
+ ## removed must be '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'.
+ valid_dir_pattern = ('^' + NIGHTLY_TESTS_WORKSPACE + '/(' +
+ '|'.join(DIR_BY_WEEKDAY) + ')')
if not re.search(valid_dir_pattern, s):
print('Trying to delete an invalid dir "{0}" (must match "{1}"), '
'please check.'.format(s, valid_dir_pattern))
@@ -192,8 +193,8 @@ def CleanOldCLs(days_to_preserve='1', dry_run=False):
ce = command_executer.GetCommandExecuter()
chromeos_root = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos')
# Find Old CLs.
- old_cls_cmd = (
- 'gerrit --raw search "owner:me status:open age:%sd"' % days_to_preserve)
+ old_cls_cmd = ('gerrit --raw search "owner:me status:open age:%sd"' %
+ days_to_preserve)
_, cls, _ = ce.ChrootRunCommandWOutput(
chromeos_root, old_cls_cmd, print_to_console=False)
# Convert any whitespaces to spaces.
@@ -210,6 +211,25 @@ def CleanOldCLs(days_to_preserve='1', dry_run=False):
chromeos_root, abandon_cls_cmd, print_to_console=False)
+def CleanChromeTelemetryTmpFiles(dry_run):
+ rv = 0
+ ce = command_executer.GetCommandExecuter()
+ tmp_dir = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', '.cache',
+ 'distfiles', 'target', 'chrome-src-internal', 'src',
+ 'tmp')
+ cmd = f'rm -fr {shlex.quote(tmp_dir)}/tmp*telemetry_Crosperf'
+ if dry_run:
+ print(f'Going to execute:\n{cmd}')
+ else:
+ rv = ce.RunCommand(cmd, print_to_console=False)
+ if rv == 0:
+ print(f'Successfully cleaned chrome tree tmp directory ' f'{tmp_dir!r} .')
+ else:
+ print(f'Some directories were not removed under chrome tree '
+ f'tmp directory {tmp_dir!r}.')
+ return rv
+
+
def Main(argv):
"""Delete nightly test data directories, tmps and test images."""
options = ProcessArguments(argv)
@@ -239,7 +259,10 @@ def Main(argv):
# Clean CLs that are not updated in last 2 weeks.
rv3 = CleanOldCLs('14', options.dry_run)
- return rv + rv2 + rv3
+ # Clean telemetry temporaries from chrome source tree inside chroot.
+ rv4 = CleanChromeTelemetryTmpFiles(options.dry_run)
+
+ return rv + rv2 + rv3 + rv4
if __name__ == '__main__':
diff --git a/binary_search_tool/common/test_setup.sh b/binary_search_tool/common/test_setup.sh
index 56b8944a..4fa1abe2 100755
--- a/binary_search_tool/common/test_setup.sh
+++ b/binary_search_tool/common/test_setup.sh
@@ -109,15 +109,17 @@ if [[ "${BISECT_MODE}" == "OBJECT_MODE" ]]; then
echo
echo "DEPLOYING"
- if [[ ${PACKAGE} == sys-kernel/chromeos-kernel-* ]]; then
- echo "/mnt/host/source/src/scripts/update_kernel.sh \
- --remote=${BISECT_REMOTE}"
- /mnt/host/source/src/scripts/update_kernel.sh --remote=${BISECT_REMOTE}
- else
- echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}"
- cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info
+
+ if [[ ${BISECT_PACKAGE} == sys-kernel/chromeos-kernel-* ]]; then
+ echo "/mnt/host/source/src/scripts/update_kernel.sh " \
+ "--remote=${BISECT_REMOTE} --board=${BISECT_BOARD}"
+ # exec the command to make sure it always exit after
+ exec /mnt/host/source/src/scripts/update_kernel.sh --remote=${BISECT_REMOTE} --board=${BISECT_BOARD}
fi
+ echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}"
+ cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info
+
deploy_status=$?
if [[ ${deploy_status} -eq 0 ]] ; then
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
index e4784a62..48987966 100755
--- a/buildbot_test_toolchains.py
+++ b/buildbot_test_toolchains.py
@@ -250,6 +250,8 @@ class ToolchainComparator(object):
self._ce.RunCommand(command)
def _CopyJson(self):
+ # Make sure a destination directory exists.
+ os.makedirs(PENDING_ARCHIVES_DIR, exist_ok=True)
# Copy json report to pending archives directory.
command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
ret = self._ce.RunCommand(command)
diff --git a/compiler_wrapper/README.md b/compiler_wrapper/README.md
index 794e1635..e55bb669 100644
--- a/compiler_wrapper/README.md
+++ b/compiler_wrapper/README.md
@@ -12,16 +12,63 @@ package, including the build script, and then
build from there without a dependency on toolchain-utils
itself.
-## Update Chrome OS
+## Testing Inside the Chroot
-Copy over sources and `build.py` to Chrome OS:
+To test updates to the wrapper locally:
+
+Run `install_compiler_wrapper.sh` to install the new wrapper in the chroot:
+```
+(chroot) ~/trunk/src/third_party/toolchain-utils/compiler_wrapper/install_compiler_wrapper.sh
+```
+
+Then perform the tests, e.g. build with the new compiler.
+
+
+## Updating the Wrapper for Chrome OS
+
+To update the wrapper for everyone, the new wrapper configuration must be copied
+into chromiumos-overlay, and new revisions of the gcc and llvm ebuilds must be
+created.
+
+Copy over sources and `build.py` to chromiumos-overlay:
```
(chroot) /mnt/host/source/src/third_party/chromiumos-overlay/sys-devel/llvm/files/update_compiler_wrapper.sh
```
+Rename chromiumos-overlay/sys-devel/llvm/llvm-${VERSION}.ebuild to the next
+revision number. For example, if the current version is
+11.0_pre394483_p20200618-r2:
+```
+(chroot) cd ~/trunk/src/third_party/chromiumos-overlay
+(chroot) git mv llvm-11.0_pre394483_p20200618-r2.ebuild llvm-11.0_pre394483_p20200618-r3.ebuild
+```
+
+Rename chromiumos-overlay/sys-devel/gcc/gcc-${VERSION}.ebuild to the next
+revision number. For example, if the current version is 4.9.2-r254:
+```
+(chroot) cd ~/trunk/src/third_party/chromiumos-overlay
+(chroot) git mv sys-devel/gcc/gcc-4.9.2-r254.ebuild sys-devel/gcc/gcc-4.9.2-r255.ebuild
+```
+
+Edit the new ebuild file and change the number at the end to match the new revision:
+```
+-# If you need to force a cros_workon uprev, change this number (you can use next
+-# uprev): 254
++# If you need to force a cros_workon uprev, change this number (you can use next
++# uprev): 255
+```
+
+Commit those changes together with the changes made by
+`update_compiler_wrapper.sh`.
+
+The changes can then be reviewed and submitted through the normal process.
+
+
+## Paths
+
`build.py` is called by these ebuilds:
-- third_party/chromiumos-overlay/sys-devel/llvm/llvm-9.0_pre361749_p20190714.ebuild
+- third_party/chromiumos-overlay/sys-devel/llvm/llvm-*.ebuild
- third_party/chromiumos-overlay/sys-devel/gcc/gcc-*.ebuild
Generated wrappers are stored here:
diff --git a/compiler_wrapper/bisect_flag.go b/compiler_wrapper/bisect_flag.go
index f07d9a63..adfa8b03 100644
--- a/compiler_wrapper/bisect_flag.go
+++ b/compiler_wrapper/bisect_flag.go
@@ -7,6 +7,7 @@ package main
import (
"errors"
"os"
+ "os/exec"
"path/filepath"
)
@@ -52,7 +53,7 @@ func calcBisectCommand(env env, cfg *config, bisectStage string, compilerCmd *co
}
}
absCompilerPath := getAbsCmdPath(env, compilerCmd)
- pythonPath, err := filepath.Abs(os.Args[0])
+ pythonPath, err := exec.LookPath(os.Args[0])
if err != nil {
return nil, err
}
diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go
index 1f0e2231..9eb951de 100644
--- a/compiler_wrapper/clang_flags.go
+++ b/compiler_wrapper/clang_flags.go
@@ -47,15 +47,7 @@ func processClangFlags(builder *commandBuilder) error {
// Use of -Qunused-arguments allows this set to be small, just those
// that clang still warns about.
unsupported := map[string]bool{
- "-mno-movbe": true,
- "-pass-exit-codes": true,
- "-Wclobbered": true,
- "-Wno-psabi": true,
- "-Wlogical-op": true,
- "-Wmissing-parameter-type": true,
- "-Wold-style-declaration": true,
- "-Woverride-init": true,
- "-Wunsafe-loop-optimizations": true,
+ "-pass-exit-codes": true,
}
unsupportedPrefixes := []string{"-Wstrict-aliasing=", "-finline-limit="}
diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go
index 2a7fbd15..a4145c22 100644
--- a/compiler_wrapper/clang_flags_test.go
+++ b/compiler_wrapper/clang_flags_test.go
@@ -212,14 +212,6 @@ func TestFilterUnsupportedClangFlags(t *testing.T) {
expectedCount int
}{
{clangX86_64, "-pass-exit-codes", 0},
- {clangX86_64, "-Wclobbered", 0},
- {clangX86_64, "-Wunsafe-loop-optimizations", 0},
- {clangX86_64, "-Wlogical-op", 0},
- {clangX86_64, "-Wmissing-parameter-type", 0},
- {clangX86_64, "-Woverride-init", 0},
- {clangX86_64, "-Wold-style-declaration", 0},
- {clangX86_64, "-Wno-psabi", 0},
- {clangX86_64, "-mno-movbe", 0},
{clangX86_64, "-Wstrict-aliasing=xyz", 0},
{clangX86_64, "-finline-limit=xyz", 0},
{"./armv7a-cros-linux-gnu-clang", "-ftrapv", 0},
diff --git a/compiler_wrapper/clang_tidy_flag.go b/compiler_wrapper/clang_tidy_flag.go
index d8bf3cb2..01387fd6 100644
--- a/compiler_wrapper/clang_tidy_flag.go
+++ b/compiler_wrapper/clang_tidy_flag.go
@@ -5,15 +5,39 @@
package main
import (
+ "encoding/json"
"fmt"
+ "io/ioutil"
+ "os"
+ "path"
"path/filepath"
"strings"
)
-func processClangTidyFlags(builder *commandBuilder) (cSrcFile string, useClangTidy bool) {
+type useTidyMode int
+
+const clangTidyCrashSubstring = "PLEASE submit a bug report"
+
+const (
+ tidyModeNone useTidyMode = iota
+ tidyModeAll
+ tidyModeTricium
+)
+
+func processClangTidyFlags(builder *commandBuilder) (cSrcFile string, clangTidyFlags []string, mode useTidyMode) {
+ builder.transformArgs(func(arg builderArg) string {
+ const prefix = "-clang-tidy-flag="
+ if !strings.HasPrefix(arg.value, prefix) {
+ return arg.value
+ }
+
+ clangTidyFlags = append(clangTidyFlags, arg.value[len(prefix):])
+ return ""
+ })
+
withTidy, _ := builder.env.getenv("WITH_TIDY")
if withTidy == "" {
- return "", false
+ return "", clangTidyFlags, tidyModeNone
}
srcFileSuffixes := []string{
".c",
@@ -24,48 +48,177 @@ func processClangTidyFlags(builder *commandBuilder) (cSrcFile string, useClangTi
".c++",
}
cSrcFile = ""
+ srcSuffix := ""
lastArg := ""
for _, arg := range builder.args {
- if hasAtLeastOneSuffix(arg.value, srcFileSuffixes) && lastArg != "-o" {
- cSrcFile = arg.value
+ if lastArg != "-o" {
+ for _, suffix := range srcFileSuffixes {
+ if strings.HasSuffix(arg.value, suffix) {
+ srcSuffix = suffix
+ cSrcFile = arg.value
+ break
+ }
+ }
}
lastArg = arg.value
}
- useClangTidy = cSrcFile != ""
- return cSrcFile, useClangTidy
-}
-func runClangTidy(env env, clangCmd *command, cSrcFile string) error {
- defaultTidyChecks := strings.Join([]string{
- "*",
- "-bugprone-narrowing-conversions",
- "-cppcoreguidelines-*",
- "-fuchsia-*",
- "-google-readability*",
- "-google-runtime-references",
- "-hicpp-*",
- "-llvm-*",
- "-misc-non-private-member-variables-in-classes",
- "-misc-unused-parameters",
- "-modernize-*",
- "-readability-*",
- }, ",")
+ if cSrcFile == "" {
+ return "", clangTidyFlags, tidyModeNone
+ }
+ if withTidy == "tricium" {
+ // Files generated from protobufs can result in _many_ clang-tidy complaints, and aren't
+ // worth linting in general. Don't.
+ if strings.HasSuffix(cSrcFile, ".pb"+srcSuffix) {
+ mode = tidyModeNone
+ } else {
+ mode = tidyModeTricium
+ }
+ } else {
+ mode = tidyModeAll
+ }
+ return cSrcFile, clangTidyFlags, mode
+}
+
+func calcClangTidyInvocation(env env, clangCmd *command, cSrcFile string, tidyFlags ...string) (*command, error) {
resourceDir, err := getClangResourceDir(env, clangCmd.Path)
if err != nil {
- return err
+ return nil, err
}
clangTidyPath := filepath.Join(filepath.Dir(clangCmd.Path), "clang-tidy")
- clangTidyCmd := &command{
- Path: clangTidyPath,
- Args: append([]string{
- "-checks=" + defaultTidyChecks,
- cSrcFile,
- "--",
- "-resource-dir=" + resourceDir,
- }, clangCmd.Args...),
+ args := append([]string{}, tidyFlags...)
+ args = append(args, cSrcFile, "--", "-resource-dir="+resourceDir)
+ args = append(args, clangCmd.Args...)
+ return &command{
+ Path: clangTidyPath,
+ Args: args,
EnvUpdates: clangCmd.EnvUpdates,
+ }, nil
+}
+
+func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir string, extraTidyFlags []string, crashArtifactsDir string) error {
+ if err := os.MkdirAll(fixesDir, 0777); err != nil {
+ return fmt.Errorf("creating fixes directory at %q: %v", fixesDir, err)
+ }
+
+ f, err := ioutil.TempFile(fixesDir, "lints-")
+ if err != nil {
+ return fmt.Errorf("making tempfile for tidy: %v", err)
+ }
+ f.Close()
+
+ // `f` is an 'anchor'; it ensures we won't create a similarly-named file in the future.
+ // Hence, we can't delete it.
+ fixesFilePath := f.Name() + ".yaml"
+ fixesMetadataPath := f.Name() + ".json"
+
+ // FIXME(gbiv): Remove `-checks=*` when testing is complete; we should defer to .clang-tidy
+ // files, which are both more expressive and more approachable than `-checks=*`.
+ extraTidyFlags = append(extraTidyFlags, "-checks=*", "--export-fixes="+fixesFilePath)
+ clangTidyCmd, err := calcClangTidyInvocation(env, clangCmd, cSrcFile, extraTidyFlags...)
+ if err != nil {
+ return fmt.Errorf("calculating tidy invocation: %v", err)
+ }
+
+ stdstreams := &strings.Builder{}
+ // Note: We pass nil as stdin as we checked before that the compiler
+ // was invoked with a source file argument.
+ exitCode, err := wrapSubprocessErrorWithSourceLoc(clangTidyCmd,
+ env.run(clangTidyCmd, nil, stdstreams, stdstreams))
+ if err != nil {
+ return err
+ }
+
+ type crashOutput struct {
+ CrashReproducerPath string `json:"crash_reproducer_path"`
+ Stdstreams string `json:"stdstreams"`
+ }
+
+ type metadata struct {
+ Args []string `json:"args"`
+ CrashOutput *crashOutput `json:"crash_output"`
+ Executable string `json:"executable"`
+ ExitCode int `json:"exit_code"`
+ LintTarget string `json:"lint_target"`
+ Stdstreams string `json:"stdstreams"`
+ Wd string `json:"wd"`
+ }
+
+ meta := &metadata{
+ Args: clangTidyCmd.Args,
+ CrashOutput: nil,
+ Executable: clangTidyCmd.Path,
+ ExitCode: exitCode,
+ LintTarget: cSrcFile,
+ Stdstreams: stdstreams.String(),
+ Wd: env.getwd(),
+ }
+
+ // Sometimes, clang-tidy crashes. Unfortunately, these don't get funnelled through the
+ // standard clang crash machinery. :(. Try to work with our own.
+ if crashArtifactsDir != "" && strings.Contains(meta.Stdstreams, clangTidyCrashSubstring) {
+ tidyCrashArtifacts := path.Join(crashArtifactsDir, "clang-tidy")
+ if err := os.MkdirAll(tidyCrashArtifacts, 0777); err != nil {
+ return fmt.Errorf("creating crash artifacts directory at %q: %v", tidyCrashArtifacts, err)
+ }
+
+ f, err := ioutil.TempFile(tidyCrashArtifacts, "crash-")
+ if err != nil {
+ return fmt.Errorf("making tempfile for crash output: %v", err)
+ }
+ f.Close()
+
+ reproCmd := &command{}
+ *reproCmd = *clangCmd
+ reproCmd.Args = append(reproCmd.Args, "-E", "-o", f.Name())
+
+ reproOut := &strings.Builder{}
+ _, err = wrapSubprocessErrorWithSourceLoc(reproCmd, env.run(reproCmd, nil, reproOut, reproOut))
+ if err != nil {
+ return fmt.Errorf("attempting to produce a clang-tidy crash reproducer: %v", err)
+ }
+ meta.CrashOutput = &crashOutput{
+ CrashReproducerPath: f.Name(),
+ Stdstreams: reproOut.String(),
+ }
+ }
+
+ f, err = os.Create(fixesMetadataPath)
+ if err != nil {
+ return fmt.Errorf("creating fixes metadata: %v", err)
+ }
+
+ if err := json.NewEncoder(f).Encode(meta); err != nil {
+ return fmt.Errorf("writing fixes metadata: %v", err)
+ }
+
+ if err := f.Close(); err != nil {
+ return fmt.Errorf("finalizing fixes metadata: %v", err)
+ }
+ return nil
+}
+
+func runClangTidy(env env, clangCmd *command, cSrcFile string, extraTidyFlags []string) error {
+ extraTidyFlags = append(extraTidyFlags,
+ "-checks="+strings.Join([]string{
+ "*",
+ "-bugprone-narrowing-conversions",
+ "-cppcoreguidelines-*",
+ "-fuchsia-*",
+ "-google-readability*",
+ "-google-runtime-references",
+ "-hicpp-*",
+ "-llvm-*",
+ "-misc-non-private-member-variables-in-classes",
+ "-misc-unused-parameters",
+ "-modernize-*",
+ "-readability-*",
+ }, ","))
+ clangTidyCmd, err := calcClangTidyInvocation(env, clangCmd, cSrcFile, extraTidyFlags...)
+ if err != nil {
+ return fmt.Errorf("calculating clang-tidy invocation: %v", err)
}
// Note: We pass nil as stdin as we checked before that the compiler
diff --git a/compiler_wrapper/clang_tidy_flag_test.go b/compiler_wrapper/clang_tidy_flag_test.go
index baf5219e..4293bb21 100644
--- a/compiler_wrapper/clang_tidy_flag_test.go
+++ b/compiler_wrapper/clang_tidy_flag_test.go
@@ -273,6 +273,176 @@ func TestPartiallyOmitGomaWithClangTidy(t *testing.T) {
})
}
+func TestTriciumClangTidyIsProperlyDetectedFromEnv(t *testing.T) {
+ withClangTidyTestContext(t, func(ctx *testContext) {
+ ctx.env = []string{"WITH_TIDY=tricium"}
+ ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+ switch ctx.cmdCount {
+ case 1:
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ }
+ return nil
+ case 2:
+ if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil {
+ return err
+ }
+
+ hasFixesFile := false
+ for _, arg := range cmd.Args {
+ if path := strings.TrimPrefix(arg, "--export-fixes="); path != arg {
+ hasFixesFile = true
+ if !strings.HasPrefix(path, ctx.cfg.triciumNitsDir+"/") {
+ t.Errorf("fixes file was %q; expected it to be in %q", path, ctx.cfg.triciumNitsDir)
+ }
+ break
+ }
+ }
+
+ if !hasFixesFile {
+ t.Error("no fixes file was provided to a tricium invocation")
+ }
+
+ return nil
+ default:
+ return nil
+ }
+ }
+ cmd := ctx.must(callCompiler(ctx, ctx.cfg,
+ ctx.newCommand(clangX86_64, mainCc)))
+ if ctx.cmdCount != 3 {
+ t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount)
+ }
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ }
+ })
+}
+
+func TestTriciumClangTidySkipsProtobufFiles(t *testing.T) {
+ withClangTidyTestContext(t, func(ctx *testContext) {
+ ctx.env = []string{"WITH_TIDY=tricium"}
+ cmd := ctx.must(callCompiler(ctx, ctx.cfg,
+ ctx.newCommand(clangX86_64, mainCc+".pb.cc")))
+ if ctx.cmdCount != 1 {
+ t.Errorf("expected tricium clang-tidy to not execute on a protobuf file")
+ }
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ }
+ })
+}
+
+func testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t *testing.T, ctx *testContext) {
+ addedFlag := "--some_clang_tidy=flag"
+ ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+ switch ctx.cmdCount {
+ case 1:
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ } else if err := verifyArgCount(cmd, 0, addedFlag); err != nil {
+ t.Error(err)
+ }
+ return nil
+ case 2:
+ if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil {
+ t.Error(err)
+ } else if verifyArgCount(cmd, 1, addedFlag); err != nil {
+ t.Error(err)
+ }
+ return nil
+ default:
+ return nil
+ }
+ }
+ cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc, "-clang-tidy-flag="+addedFlag)))
+ if ctx.cmdCount != 3 {
+ t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount)
+ }
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ }
+}
+
+func TestClangTidyFiltersClangTidySpecificFlagsForTricium(t *testing.T) {
+ withClangTidyTestContext(t, func(ctx *testContext) {
+ ctx.env = []string{"WITH_TIDY=tricium"}
+ testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t, ctx)
+ })
+}
+
+func TestClangTidyFiltersClangTidySpecificFlags(t *testing.T) {
+ withClangTidyTestContext(t, func(ctx *testContext) {
+ testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t, ctx)
+ })
+}
+
+func TestClangTidyFlagsAreFilteredFromGccInvocations(t *testing.T) {
+ withTestContext(t, func(ctx *testContext) {
+ cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc, "-clang-tidy-flag=--foo")))
+ if err := verifyArgCount(cmd, 0, ".*--foo.*"); err != nil {
+ t.Error(err)
+ }
+ })
+}
+
+func TestTriciumReportsClangTidyCrashesGracefully(t *testing.T) {
+ withClangTidyTestContext(t, func(ctx *testContext) {
+ ctx.env = []string{"WITH_TIDY=tricium"}
+ ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+ switch ctx.cmdCount {
+ case 1:
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ }
+ return nil
+ case 2:
+ if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil {
+ return err
+ }
+
+ if _, err := io.WriteString(stdout, clangTidyCrashSubstring); err != nil {
+ return err
+ }
+ return nil
+ case 3:
+ if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+ t.Error(err)
+ }
+
+ args := cmd.Args
+ if len(args) < 3 {
+ t.Errorf("insufficient number of args provided; got %d; want at least 3", len(args))
+ return nil
+ }
+
+ lastArgs := args[len(args)-3:]
+ eArg, oArg, outFileArg := lastArgs[0], lastArgs[1], lastArgs[2]
+ if eArg != "-E" {
+ t.Errorf("got eArg=%q; wanted -E", eArg)
+ }
+
+ if oArg != "-o" {
+ t.Errorf("got oArg=%q; wanted -o", oArg)
+ }
+
+ wantPrefix := path.Join(ctx.cfg.crashArtifactsDir, "clang-tidy")
+ if !strings.HasPrefix(outFileArg, wantPrefix) {
+ t.Errorf("got out file %q; wanted one starting with %q", outFileArg, wantPrefix)
+ }
+
+ return nil
+ default:
+ return nil
+ }
+ }
+ ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc)))
+ if ctx.cmdCount != 4 {
+ t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount)
+ }
+ })
+}
+
func withClangTidyTestContext(t *testing.T, work func(ctx *testContext)) {
withTestContext(t, func(ctx *testContext) {
ctx.env = []string{"WITH_TIDY=1"}
diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go
index 4c3db14a..2b9459e4 100644
--- a/compiler_wrapper/compiler_wrapper.go
+++ b/compiler_wrapper/compiler_wrapper.go
@@ -80,6 +80,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int
case clangType:
mainBuilder.addPreUserArgs(mainBuilder.cfg.clangFlags...)
mainBuilder.addPreUserArgs(mainBuilder.cfg.commonFlags...)
+ mainBuilder.addPostUserArgs(mainBuilder.cfg.clangPostFlags...)
if _, err := processGomaCccFlags(mainBuilder); err != nil {
return 0, err
}
@@ -89,40 +90,55 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int
default:
return 0, newErrorwithSourceLocf("unsupported compiler: %s", mainBuilder.target.compiler)
}
- } else if mainBuilder.target.compilerType == clangType {
- cSrcFile, useClangTidy := processClangTidyFlags(mainBuilder)
- err := prepareClangCommand(mainBuilder)
- if err != nil {
- return 0, err
- }
- allowCCache := true
- if useClangTidy {
- allowCCache = false
- clangCmdWithoutGomaAndCCache := mainBuilder.build()
- if err := runClangTidy(env, clangCmdWithoutGomaAndCCache, cSrcFile); err != nil {
- return 0, err
- }
- }
- if err := processGomaCCacheFlags(allowCCache, mainBuilder); err != nil {
- return 0, err
- }
- compilerCmd = mainBuilder.build()
} else {
- if clangSyntax {
- allowCCache := false
- clangCmd, err := calcClangCommand(allowCCache, mainBuilder.clone())
+ cSrcFile, tidyFlags, tidyMode := processClangTidyFlags(mainBuilder)
+ if mainBuilder.target.compilerType == clangType {
+ err := prepareClangCommand(mainBuilder)
if err != nil {
return 0, err
}
- gccCmd, err := calcGccCommand(mainBuilder)
+ allowCCache := true
+ if tidyMode != tidyModeNone {
+ allowCCache = false
+ clangCmdWithoutGomaAndCCache := mainBuilder.build()
+ var err error
+ switch tidyMode {
+ case tidyModeTricium:
+ if cfg.triciumNitsDir == "" {
+ return 0, newErrorwithSourceLocf("tricium linting was requested, but no nits directory is configured")
+ }
+ err = runClangTidyForTricium(env, clangCmdWithoutGomaAndCCache, cSrcFile, cfg.triciumNitsDir, tidyFlags, cfg.crashArtifactsDir)
+ case tidyModeAll:
+ err = runClangTidy(env, clangCmdWithoutGomaAndCCache, cSrcFile, tidyFlags)
+ default:
+ panic(fmt.Sprintf("Unknown tidy mode: %v", tidyMode))
+ }
+
+ if err != nil {
+ return 0, err
+ }
+ }
+ if err := processGomaCCacheFlags(allowCCache, mainBuilder); err != nil {
+ return 0, err
+ }
+ compilerCmd = mainBuilder.build()
+ } else {
+ if clangSyntax {
+ allowCCache := false
+ clangCmd, err := calcClangCommand(allowCCache, mainBuilder.clone())
+ if err != nil {
+ return 0, err
+ }
+ gccCmd, err := calcGccCommand(mainBuilder)
+ if err != nil {
+ return 0, err
+ }
+ return checkClangSyntax(env, clangCmd, gccCmd)
+ }
+ compilerCmd, err = calcGccCommand(mainBuilder)
if err != nil {
return 0, err
}
- return checkClangSyntax(env, clangCmd, gccCmd)
- }
- compilerCmd, err = calcGccCommand(mainBuilder)
- if err != nil {
- return 0, err
}
}
rusageLogfileName := getRusageLogFilename(env)
@@ -167,6 +183,9 @@ func prepareClangCommand(builder *commandBuilder) (err error) {
processSysrootFlag(builder)
}
builder.addPreUserArgs(builder.cfg.clangFlags...)
+ if builder.cfg.crashArtifactsDir != "" {
+ builder.addPreUserArgs("-fcrash-diagnostics-dir=" + builder.cfg.crashArtifactsDir)
+ }
builder.addPostUserArgs(builder.cfg.clangPostFlags...)
calcCommonPreUserArgs(builder)
return processClangFlags(builder)
diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go
index a122b5a2..194ab579 100644
--- a/compiler_wrapper/config.go
+++ b/compiler_wrapper/config.go
@@ -29,6 +29,10 @@ type config struct {
rootRelPath string
// Directory to store errors that were prevented with -Wno-error.
newWarningsDir string
+ // Directory to store nits in when using `WITH_TIDY=tricium`.
+ triciumNitsDir string
+ // Directory to store crash artifacts in.
+ crashArtifactsDir string
// Version. Only used for printing via -print-cmd.
version string
}
@@ -92,6 +96,7 @@ func getConfig(configName string, useCCache bool, useLlvmNext bool, version stri
cfg.useLlvmNext = useLlvmNext
if useLlvmNext {
cfg.clangFlags = append(cfg.clangFlags, llvmNextFlags...)
+ cfg.clangPostFlags = append(cfg.clangPostFlags, llvmNextPostFlags...)
}
cfg.version = version
return &cfg, nil
@@ -119,9 +124,10 @@ var crosHardenedConfig = &config{
// Disable "-faddrsig" since it produces object files that strip doesn't understand, chromium:915742.
// Pass "-fcommon" till the packages are fixed to work with new clang default
// "-fno-common", crbug.com/1060413.
+ // crbug.com/1103065: -grecord-gcc-switches pollutes the Goma cache;
+ // removed that flag for now.
clangFlags: []string{
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -132,11 +138,14 @@ var crosHardenedConfig = &config{
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
},
clangPostFlags: []string{
"-Wno-implicit-int-float-conversion",
},
- newWarningsDir: "/tmp/fatal_clang_warnings",
+ newWarningsDir: "/tmp/fatal_clang_warnings",
+ triciumNitsDir: "/tmp/linting_output/clang-tidy",
+ crashArtifactsDir: "/tmp/clang_crash_diagnostics",
}
// Flags to be added to non-hardened toolchain.
@@ -161,11 +170,14 @@ var crosNonHardenedConfig = &config{
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
},
clangPostFlags: []string{
"-Wno-implicit-int-float-conversion",
},
- newWarningsDir: "/tmp/fatal_clang_warnings",
+ newWarningsDir: "/tmp/fatal_clang_warnings",
+ triciumNitsDir: "/tmp/linting_output/clang-tidy",
+ crashArtifactsDir: "/tmp/clang_crash_diagnostics",
}
// Flags to be added to host toolchain.
@@ -182,9 +194,10 @@ var crosHostConfig = &config{
// Temporarily add no-unknown-warning-option to deal with old clang versions.
// Pass "-fcommon" till the packages are fixed to work with new clang default
// "-fno-common", crbug.com/1060413.
+ // crbug.com/1103065: -grecord-gcc-switches pollutes the Goma cache;
+ // removed that flag for now.
clangFlags: []string{
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -195,20 +208,25 @@ var crosHostConfig = &config{
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
},
clangPostFlags: []string{
"-Wno-implicit-int-float-conversion",
},
- newWarningsDir: "/tmp/fatal_clang_warnings",
+ newWarningsDir: "/tmp/fatal_clang_warnings",
+ triciumNitsDir: "/tmp/linting_output/clang-tidy",
+ crashArtifactsDir: "/tmp/clang_crash_diagnostics",
}
var androidConfig = &config{
- isHostWrapper: false,
- isAndroidWrapper: true,
- rootRelPath: "./",
- commonFlags: []string{},
- gccFlags: []string{},
- clangFlags: []string{},
- clangPostFlags: []string{},
- newWarningsDir: "",
+ isHostWrapper: false,
+ isAndroidWrapper: true,
+ rootRelPath: "./",
+ commonFlags: []string{},
+ gccFlags: []string{},
+ clangFlags: []string{},
+ clangPostFlags: []string{},
+ newWarningsDir: "",
+ triciumNitsDir: "",
+ crashArtifactsDir: "",
}
diff --git a/compiler_wrapper/cros_llvm_next_flags.go b/compiler_wrapper/cros_llvm_next_flags.go
index 6cd7cd2a..f5823fbf 100644
--- a/compiler_wrapper/cros_llvm_next_flags.go
+++ b/compiler_wrapper/cros_llvm_next_flags.go
@@ -13,6 +13,11 @@ package main
// tag is set.
// TODO: Enable test in config_test.go, once we have new llvm-next flags.
-var llvmNextFlags = []string{}
+var llvmNextFlags = []string{
+ "-Wno-compound-token-split-by-macro",
+}
-var llvmNextPostFlags = []string{}
+var llvmNextPostFlags = []string{
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+}
diff --git a/compiler_wrapper/update_compiler_wrapper.sh b/compiler_wrapper/install_compiler_wrapper.sh
index 479b112c..479b112c 100755
--- a/compiler_wrapper/update_compiler_wrapper.sh
+++ b/compiler_wrapper/install_compiler_wrapper.sh
diff --git a/compiler_wrapper/libc_exec.go b/compiler_wrapper/libc_exec.go
index f8db9d86..d9867733 100644
--- a/compiler_wrapper/libc_exec.go
+++ b/compiler_wrapper/libc_exec.go
@@ -7,15 +7,31 @@
package main
// #include <errno.h>
+// #include <stdio.h>
// #include <stdlib.h>
// #include <string.h>
// #include <unistd.h>
+// #include <sys/types.h>
+// #include <sys/wait.h>
//
// int libc_exec(const char *pathname, char *const argv[], char *const envp[]) {
-// if (execve(pathname, argv, envp) != 0) {
+// // Since fork() brings us to one thread, we can only use async-signal-safe funcs below.
+// pid_t pid = fork();
+// if (pid == 0) {
+// execve(pathname, argv, envp);
+// fprintf(stderr, "exec failed (errno: %d)\n", errno);
+// _exit(1);
+// }
+// if (pid < 0) {
+// return errno;
+// }
+//
+// int wstatus;
+// pid_t waited = waitpid(pid, &wstatus, 0);
+// if (waited == -1) {
// return errno;
// }
-// return 0;
+// exit(WEXITSTATUS(wstatus));
//}
import "C"
import (
@@ -29,7 +45,7 @@ import (
// Note that this changes the go binary to be a dynamically linked one.
// See crbug.com/1000863 for details.
// To use this version of exec, please add '-tags libc_exec' when building Go binary.
-// Without the tags, libc_exec.go will be used.
+// Without the tags, libc_exec.go will not be used.
func execCmd(env env, cmd *command) error {
freeList := []unsafe.Pointer{}
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
index 606fecda..cc55bc4f 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
@@ -25,7 +25,6 @@
"/tmp/sysroot_bisect",
"/tmp/stable/clang",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -36,6 +35,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
],
@@ -73,7 +74,6 @@
"someBisectDir",
"/tmp/stable/clang",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -84,6 +84,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
],
@@ -124,7 +126,6 @@
"someBisectDir",
"/tmp/stable/clang",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -135,6 +136,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
],
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json
index 7f45584a..5a59277c 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json
@@ -16,7 +16,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -27,6 +26,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -51,7 +52,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -62,6 +62,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -86,7 +88,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -97,6 +98,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -121,7 +124,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -132,6 +134,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -156,7 +160,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -167,6 +170,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -191,7 +196,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -202,6 +206,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -226,7 +232,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -237,6 +242,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -261,7 +268,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -272,6 +278,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -296,7 +304,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -307,6 +314,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json
index afb0329e..44b2fb91 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json
@@ -15,7 +15,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -26,6 +25,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json
index 05b0ff96..0b951f3e 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json
@@ -15,7 +15,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -26,6 +25,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -49,7 +50,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -60,6 +60,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -83,7 +85,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -94,6 +95,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -117,7 +120,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -128,6 +130,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -151,7 +155,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -162,6 +165,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -185,7 +190,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -196,6 +200,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -219,7 +225,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -230,6 +235,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -253,7 +260,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -264,6 +270,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -287,7 +295,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -298,6 +305,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json
index f7924a06..69fb4410 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json
@@ -15,7 +15,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -26,6 +25,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -52,7 +53,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -63,6 +63,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -89,7 +91,6 @@
"path": "/tmp/stable/clang++",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -100,6 +101,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -126,7 +129,6 @@
"path": "somepath/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -137,6 +139,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -170,7 +174,6 @@
"path": "/somedir/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -181,6 +184,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-resource-dir=someResourceDir",
"--gcc-toolchain=/usr",
"main.cc",
@@ -219,7 +224,6 @@
"path": "/somedir/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -230,6 +234,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-resource-dir=someResourceDir",
"--gcc-toolchain=/usr",
"main.cc",
@@ -268,7 +274,6 @@
"path": "/somedir/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -279,6 +284,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-resource-dir=someResourceDir",
"--gcc-toolchain=/usr",
"main.cc",
@@ -307,7 +314,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -318,6 +324,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -341,7 +349,6 @@
"path": "/tmp/stable/a/b/c/d/e/f/g/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -352,6 +359,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -375,7 +384,6 @@
"path": "/tmp/stable/a/b/c/d/e/f/g/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -386,6 +394,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -409,7 +419,6 @@
"path": "/tmp/stable/somedir/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -420,6 +429,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -446,7 +457,6 @@
"path": "/tmp/stable/pathenv/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -457,6 +467,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json
index 11ebc821..93e566fe 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json
@@ -17,7 +17,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -28,6 +27,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -54,7 +55,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -65,6 +65,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -91,7 +93,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -102,6 +103,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -128,7 +131,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -139,6 +141,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -164,7 +168,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -175,6 +178,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fno-experimental-new-pass-manager",
"-fsanitize=fuzzer",
"main.cc",
@@ -202,7 +207,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -213,6 +217,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fno-experimental-new-pass-manager",
"-fsanitize=address",
"-fprofile-instr-generate",
@@ -240,7 +246,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -251,6 +256,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=address",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -276,7 +283,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -287,6 +293,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fprofile-instr-generate",
"main.cc",
"-Wno-implicit-int-float-conversion"
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json
index 7a724c88..41143bf0 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json
@@ -26,7 +26,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -37,6 +36,16 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
+ "-mno-movbe",
+ "-Wclobbered",
+ "-Wno-psabi",
+ "-Wlogical-op",
+ "-Wmissing-parameter-type",
+ "-Wold-style-declaration",
+ "-Woverride-init",
+ "-Wunsafe-loop-optimizations",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -61,7 +70,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -72,6 +80,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-#warnings",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -97,7 +107,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -108,6 +117,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-error=uninitialized",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -133,7 +144,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -144,6 +154,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-error=unused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -169,7 +181,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -180,6 +191,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-unused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -205,7 +218,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -216,6 +228,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wunused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion"
@@ -241,7 +255,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -252,6 +265,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-someflag",
"main.cc",
"-Wno-implicit-int-float-conversion"
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json
index b4cd44c2..c4f86b01 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json
@@ -31,7 +31,6 @@
"--",
"-resource-dir=someResourceDir",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -42,6 +41,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -52,7 +53,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -63,6 +63,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -103,7 +105,6 @@
"--",
"-resource-dir=someResourceDir",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -114,6 +115,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -124,7 +127,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -135,6 +137,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -177,7 +181,6 @@
"--",
"-resource-dir=someResourceDir",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -188,6 +191,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -201,7 +206,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -212,6 +216,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -255,7 +261,6 @@
"--",
"-resource-dir=someResourceDir",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -266,6 +271,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -276,7 +283,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -287,6 +293,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json
index 66e58dc9..272b4220 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json
@@ -18,7 +18,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -29,6 +28,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -55,7 +56,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -66,6 +66,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -78,7 +80,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -89,6 +90,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
"-Wno-error",
@@ -119,7 +122,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -130,6 +132,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion"
]
@@ -142,7 +146,6 @@
"path": "/tmp/stable/clang",
"args": [
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-fuse-ld=lld",
@@ -153,6 +156,8 @@
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
"-Wno-unknown-warning-option",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
"-Wno-error",
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
index 5b95cdb7..97dec849 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
@@ -27,7 +27,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -38,6 +37,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -45,6 +46,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -88,7 +90,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -99,6 +100,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -106,6 +109,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -152,7 +156,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -163,6 +166,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -170,6 +175,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json
index aa083d70..6223ed91 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json
@@ -18,7 +18,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -29,6 +28,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -37,6 +38,7 @@
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -69,7 +71,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-eabi",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -80,6 +81,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -88,6 +91,7 @@
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-eabi"
@@ -120,7 +124,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-win-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -131,6 +134,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -139,6 +144,7 @@
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-win-gnu"
@@ -171,7 +177,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7m-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -182,6 +187,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -222,7 +229,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7m-cros-eabi",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -233,6 +239,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -273,7 +281,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7m-cros-win-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -284,6 +291,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -324,7 +333,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv8m-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -335,6 +343,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -375,7 +385,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv8m-cros-eabi",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -386,6 +395,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -426,7 +437,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv8m-cros-win-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -437,6 +447,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json
index 257c0a05..717681c4 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json
@@ -17,7 +17,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -28,6 +27,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -35,6 +36,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -66,7 +68,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-eabi",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -77,6 +78,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -84,6 +87,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-eabi"
@@ -115,7 +119,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-win-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -126,6 +129,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -133,6 +138,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-win-gnu"
@@ -164,7 +170,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7m-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -175,6 +180,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -213,7 +220,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7m-cros-eabi",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -224,6 +230,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -262,7 +270,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7m-cros-win-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -273,6 +280,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -311,7 +320,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv8m-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -322,6 +330,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -360,7 +370,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv8m-cros-eabi",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -371,6 +380,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -409,7 +420,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv8m-cros-win-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -420,6 +430,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json
index b9edc49d..6940fcaf 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json
@@ -17,7 +17,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -28,6 +27,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -35,6 +36,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -69,7 +71,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -80,6 +81,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -87,6 +90,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -121,7 +125,6 @@
"../../usr/bin/clang++",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -132,6 +135,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -139,6 +144,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -173,7 +179,6 @@
"somepath/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -184,6 +189,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -191,6 +198,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -232,7 +240,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -243,6 +250,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -252,6 +261,7 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -296,7 +306,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -307,6 +316,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -316,6 +327,7 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -355,7 +367,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -366,6 +377,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -375,6 +388,7 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -409,7 +423,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -420,6 +433,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -427,6 +442,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -458,7 +474,6 @@
"a/b/usr/bin/clang",
"--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -469,6 +484,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -476,6 +493,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -507,7 +525,6 @@
"a/b/usr/bin/clang",
"--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -518,6 +535,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -525,6 +544,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -556,7 +576,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -567,6 +586,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -574,6 +595,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -608,7 +630,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -619,6 +640,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -626,6 +649,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json
index 2d70dfbf..acf6c24c 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json
@@ -19,7 +19,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -30,6 +29,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -37,6 +38,7 @@
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -70,7 +72,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -81,6 +82,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -88,6 +91,7 @@
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -121,7 +125,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -132,6 +135,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -139,6 +144,7 @@
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -172,7 +178,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -183,6 +188,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -190,6 +197,7 @@
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -222,7 +230,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -233,6 +240,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -241,6 +250,7 @@
"-fsanitize=fuzzer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -274,7 +284,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -285,6 +294,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -294,6 +305,7 @@
"-fprofile-instr-generate",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -326,7 +338,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -337,6 +348,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -344,6 +357,7 @@
"-fsanitize=address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -376,7 +390,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -387,6 +400,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -395,6 +410,7 @@
"-fprofile-instr-generate",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json
index 50f71a6f..26f5a665 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json
@@ -28,7 +28,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -39,13 +38,24 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
"-D_FORTIFY_SOURCE=2",
"-fno-omit-frame-pointer",
+ "-mno-movbe",
+ "-Wclobbered",
+ "-Wno-psabi",
+ "-Wlogical-op",
+ "-Wmissing-parameter-type",
+ "-Wold-style-declaration",
+ "-Woverride-init",
+ "-Wunsafe-loop-optimizations",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -78,7 +88,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -89,6 +98,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -97,6 +108,7 @@
"-Wno-#warnings",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -129,7 +141,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -140,6 +151,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -148,6 +161,7 @@
"-Wno-error=uninitialized",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -180,7 +194,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -191,6 +204,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -199,6 +214,7 @@
"-Wno-error=unused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -231,7 +247,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -242,6 +257,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -250,6 +267,7 @@
"-Wno-unused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -282,7 +300,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -293,6 +310,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -301,6 +320,7 @@
"-Wunused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -333,7 +353,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -344,6 +363,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -352,6 +373,7 @@
"-someflag",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json
index b690e3ed..247f207c 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json
@@ -52,7 +52,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -63,6 +62,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -70,6 +71,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -104,7 +106,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -115,6 +116,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -122,6 +125,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -149,7 +153,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -160,11 +163,14 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-D_FORTIFY_SOURCE=2",
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -197,7 +203,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -208,12 +213,15 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-D_FORTIFY_SOURCE=2",
"-fno-omit-frame-pointer",
"-fno-stack-protector",
"-D__KERNEL__",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -246,7 +254,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/armv7a-cros-linux-gnueabihf",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -257,6 +264,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-D_FORTIFY_SOURCE=2",
"-mthumb",
"-fno-stack-protector",
@@ -294,7 +303,6 @@
"args": [
"../../usr/bin/clang",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -305,6 +313,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -313,6 +323,7 @@
"--sysroot=xyz",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json
index 26abab87..219a8b2f 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json
@@ -32,7 +32,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -43,6 +42,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -50,6 +51,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -62,7 +64,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -73,6 +74,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -80,6 +83,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -122,7 +126,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -133,6 +136,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -140,6 +145,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -153,7 +159,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -164,6 +169,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -171,6 +178,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -215,7 +223,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -226,6 +233,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -233,6 +242,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -249,7 +259,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -260,6 +269,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -267,6 +278,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -312,7 +324,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -323,6 +334,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -330,6 +343,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -343,7 +357,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -354,6 +367,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -361,6 +376,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json
index 61a63a37..10e485da 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json
@@ -20,7 +20,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -31,6 +30,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -38,6 +39,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -72,7 +74,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -83,6 +84,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -90,6 +93,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -110,7 +114,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -121,6 +124,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -128,6 +133,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -166,7 +172,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -177,6 +182,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -184,6 +191,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -204,7 +212,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -215,6 +222,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -222,6 +231,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json
index a37725c0..ea804d97 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json
@@ -17,7 +17,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -28,6 +27,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -35,6 +36,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -90,7 +92,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -101,6 +102,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -108,6 +111,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -158,7 +162,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -169,6 +172,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -176,6 +181,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -210,7 +216,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -221,6 +226,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -228,6 +235,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
index 5b95cdb7..b8e0e462 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
@@ -27,7 +27,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -38,6 +37,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -45,6 +47,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -88,7 +93,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -99,6 +103,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -106,6 +113,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -152,7 +162,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -163,6 +172,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -170,6 +182,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json
index b9edc49d..9780356e 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json
@@ -17,7 +17,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -28,6 +27,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -35,6 +37,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -69,7 +74,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -80,6 +84,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -87,6 +94,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -121,7 +131,6 @@
"../../usr/bin/clang++",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -132,6 +141,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -139,6 +151,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -173,7 +188,6 @@
"somepath/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -184,6 +198,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -191,6 +208,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -232,7 +252,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -243,6 +262,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -252,6 +274,9 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -296,7 +321,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -307,6 +331,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -316,6 +343,9 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -355,7 +385,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -366,6 +395,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -375,6 +407,9 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -409,7 +444,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -420,6 +454,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -427,6 +464,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -458,7 +498,6 @@
"a/b/usr/bin/clang",
"--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -469,6 +508,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -476,6 +518,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -507,7 +552,6 @@
"a/b/usr/bin/clang",
"--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -518,6 +562,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -525,6 +572,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -556,7 +606,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -567,6 +616,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -574,6 +626,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -608,7 +663,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -619,6 +673,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -626,6 +683,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json
index 26abab87..06e114cb 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json
@@ -32,7 +32,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -43,6 +42,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -50,6 +52,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -62,7 +67,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -73,6 +77,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -80,6 +87,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -122,7 +132,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -133,6 +142,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -140,6 +152,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -153,7 +168,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -164,6 +178,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -171,6 +188,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -215,7 +235,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -226,6 +245,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -233,6 +255,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -249,7 +274,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -260,6 +284,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -267,6 +294,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -312,7 +342,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -323,6 +352,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -330,6 +362,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -343,7 +378,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -354,6 +388,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -361,6 +398,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json
index 61a63a37..cd460a79 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json
@@ -20,7 +20,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -31,6 +30,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -38,6 +40,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -72,7 +77,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -83,6 +87,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -90,6 +97,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -110,7 +120,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -121,6 +130,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -128,6 +140,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -166,7 +181,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -177,6 +191,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -184,6 +201,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -204,7 +224,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -215,6 +234,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -222,6 +244,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json
index a37725c0..c7f619d9 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json
@@ -17,7 +17,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -28,6 +27,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -35,6 +37,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -90,7 +95,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -101,6 +105,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -108,6 +115,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -158,7 +168,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -169,6 +178,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -176,6 +188,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -210,7 +225,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -221,6 +235,9 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-Wno-compound-token-split-by-macro",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -228,6 +245,9 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-Wno-compound-token-split-by-space",
+ "-Wno-string-concatenation",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
index e9bc3661..af4e3ff5 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
@@ -26,7 +26,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -37,6 +36,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -44,6 +45,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -83,7 +85,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -94,6 +95,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -101,6 +104,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -143,7 +147,6 @@
"/usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -154,6 +157,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -161,6 +166,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json
index c21d6198..50423288 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json
@@ -16,7 +16,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -27,6 +26,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -34,6 +35,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -62,7 +64,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -73,6 +74,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -80,6 +83,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -108,7 +112,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -119,6 +122,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -126,6 +131,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -154,7 +160,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -165,6 +170,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -172,6 +179,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -207,7 +215,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -218,6 +225,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -227,6 +236,7 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -266,7 +276,6 @@
"/somedir/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -277,6 +286,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -286,6 +297,7 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -324,7 +336,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -335,6 +346,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -344,6 +357,7 @@
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -372,7 +386,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -383,6 +396,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -390,6 +405,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -415,7 +431,6 @@
"args": [
"--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -426,6 +441,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -433,6 +450,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -458,7 +476,6 @@
"args": [
"--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -469,6 +486,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -476,6 +495,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -501,7 +521,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -512,6 +531,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -519,6 +540,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -547,7 +569,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -558,6 +579,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -565,6 +588,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json
index 26abab87..219a8b2f 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json
@@ -32,7 +32,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -43,6 +42,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -50,6 +51,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -62,7 +64,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -73,6 +74,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -80,6 +83,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -122,7 +126,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -133,6 +136,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -140,6 +145,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -153,7 +159,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -164,6 +169,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -171,6 +178,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -215,7 +223,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -226,6 +233,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -233,6 +242,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -249,7 +259,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -260,6 +269,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -267,6 +278,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -312,7 +324,6 @@
"-resource-dir=someResourceDir",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -323,6 +334,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -330,6 +343,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -343,7 +357,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -354,6 +367,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -361,6 +376,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json
index ce35f311..a7fa8525 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json
@@ -19,7 +19,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -30,6 +29,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -37,6 +38,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -65,7 +67,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -76,6 +77,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -83,6 +86,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -97,7 +101,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -108,6 +111,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -115,6 +120,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -147,7 +153,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -158,6 +163,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -165,6 +172,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -179,7 +187,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -190,6 +197,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -197,6 +206,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json
index f716dde3..bafca635 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json
@@ -17,7 +17,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -28,6 +27,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -35,6 +36,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -85,7 +87,6 @@
"../../usr/bin/clang",
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -96,6 +97,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -103,6 +106,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -153,7 +157,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -164,6 +167,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -171,6 +176,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -205,7 +211,6 @@
"args": [
"--sysroot=/usr/x86_64-cros-linux-gnu",
"-Qunused-arguments",
- "-grecord-gcc-switches",
"-fno-addrsig",
"-fcommon",
"-Wno-tautological-constant-compare",
@@ -216,6 +221,8 @@
"-fuse-ld=lld",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fstack-protector-strong",
"-fPIE",
"-pie",
@@ -223,6 +230,7 @@
"-fno-omit-frame-pointer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
index ce83cd72..8b03c092 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
@@ -34,8 +34,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -86,8 +89,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -141,8 +147,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json
index db5bea28..5d72756d 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json
@@ -25,9 +25,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -67,9 +70,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-eabi"
@@ -109,9 +115,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-win-gnu"
@@ -151,6 +160,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"-ftrapv",
"main.cc",
@@ -194,6 +205,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
@@ -236,6 +249,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"-ftrapv",
"main.cc",
@@ -279,6 +294,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"-ftrapv",
"main.cc",
@@ -322,6 +339,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-ftrapv",
"main.cc",
"-Wno-implicit-int-float-conversion",
@@ -364,6 +383,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"-ftrapv",
"main.cc",
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json
index 8b5d7088..382b7097 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json
@@ -24,8 +24,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -64,8 +67,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-eabi"
@@ -104,8 +110,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-win-gnu"
@@ -144,6 +153,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"main.cc",
"-Wno-implicit-int-float-conversion",
@@ -185,6 +196,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
"-B../../bin",
@@ -225,6 +238,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"main.cc",
"-Wno-implicit-int-float-conversion",
@@ -266,6 +281,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"main.cc",
"-Wno-implicit-int-float-conversion",
@@ -307,6 +324,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
"-B../../bin",
@@ -347,6 +366,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"main.cc",
"-Wno-implicit-int-float-conversion",
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json
index 418f9d62..cadfd1fe 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json
@@ -24,8 +24,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -67,8 +70,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -110,8 +116,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -153,8 +162,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -203,10 +215,13 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-resource-dir=someResourceDir",
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -258,10 +273,13 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-resource-dir=someResourceDir",
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -308,10 +326,13 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-resource-dir=someResourceDir",
"--gcc-toolchain=/usr",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -353,8 +374,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -393,8 +417,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -433,8 +460,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-Ba/b/bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -473,8 +503,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -516,8 +549,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json
index 54a3268c..8786db27 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json
@@ -26,9 +26,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -69,9 +72,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -112,9 +118,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -155,9 +164,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=kernel-address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -197,10 +209,13 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fno-experimental-new-pass-manager",
"-fsanitize=fuzzer",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -241,11 +256,14 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fno-experimental-new-pass-manager",
"-fsanitize=address",
"-fprofile-instr-generate",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -285,9 +303,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fsanitize=address",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -327,9 +348,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fprofile-instr-generate",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json
index 4a30d74e..98ca5a67 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json
@@ -35,8 +35,19 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
+ "-mno-movbe",
+ "-Wclobbered",
+ "-Wno-psabi",
+ "-Wlogical-op",
+ "-Wmissing-parameter-type",
+ "-Wold-style-declaration",
+ "-Woverride-init",
+ "-Wunsafe-loop-optimizations",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -76,9 +87,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-#warnings",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -118,9 +132,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-error=uninitialized",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -160,9 +177,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-error=unused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -202,9 +222,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wno-unused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -244,9 +267,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-Wunused-variable",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -286,9 +312,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-someflag",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json
index 8b5c8b90..d39e2b0a 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json
@@ -55,8 +55,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -98,8 +101,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -134,8 +140,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -175,10 +184,13 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-fno-stack-protector",
"-D__KERNEL__",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -218,6 +230,8 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"-mthumb",
"-fno-stack-protector",
"-D__KERNEL__",
@@ -261,9 +275,12 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"--sysroot=xyz",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json
index cf30514c..ce9659a2 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json
@@ -39,8 +39,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -60,8 +63,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -111,8 +117,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -133,8 +142,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -186,8 +198,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -211,8 +226,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -265,8 +283,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -287,8 +308,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json
index 7aad322f..61bff7d7 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json
@@ -27,8 +27,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -70,8 +73,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -99,8 +105,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -146,8 +155,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu"
@@ -175,8 +187,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json
index 76d81fa9..244e8fad 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json
@@ -24,8 +24,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -84,8 +87,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -139,8 +145,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
@@ -182,8 +191,11 @@
"-static-libgcc",
"-Wno-final-dtor-non-final-class",
"-Werror=poison-system-directories",
+ "-fexperimental-new-pass-manager",
+ "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics",
"main.cc",
"-Wno-implicit-int-float-conversion",
+ "-mno-movbe",
"-B../../bin",
"-target",
"x86_64-cros-linux-gnu",
diff --git a/compiler_wrapper/testutil_test.go b/compiler_wrapper/testutil_test.go
index 5d8ef920..21b7169d 100644
--- a/compiler_wrapper/testutil_test.go
+++ b/compiler_wrapper/testutil_test.go
@@ -141,6 +141,8 @@ func (ctx *testContext) mustFail(exitCode int) string {
func (ctx *testContext) updateConfig(cfg *config) {
*ctx.cfg = *cfg
ctx.cfg.newWarningsDir = filepath.Join(ctx.tempDir, "fatal_clang_warnings")
+ ctx.cfg.triciumNitsDir = filepath.Join(ctx.tempDir, "tricium_nits")
+ ctx.cfg.crashArtifactsDir = filepath.Join(ctx.tempDir, "clang_crash_diagnostics")
}
func (ctx *testContext) newCommand(path string, args ...string) *command {
diff --git a/cros_utils/buildbot_json.py b/cros_utils/buildbot_json.py
deleted file mode 100755
index 08a8ae05..00000000
--- a/cros_utils/buildbot_json.py
+++ /dev/null
@@ -1,1534 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# NOTE: This file is NOT under GPL. See above.
-
-"""Queries buildbot through the json interface.
-"""
-
-from __future__ import print_function
-
-__author__ = 'maruel@chromium.org'
-__version__ = '1.2'
-
-import code
-import datetime
-import functools
-import json
-
-# Pylint recommends we use "from chromite.lib import cros_logging as logging".
-# Chromite specific policy message, we want to keep using the standard logging.
-# pylint: disable=cros-logging-import
-import logging
-
-# pylint: disable=deprecated-module
-import optparse
-
-import time
-import sys
-import urllib.error
-import urllib.parse
-import urllib.request
-
-try:
- from natsort import natsorted
-except ImportError:
- # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted
- # after "vm7". Defaults to normal sorting.
- natsorted = sorted
-
-# These values are buildbot constants used for Build and BuildStep.
-# This line was copied from master/buildbot/status/builder.py.
-SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = list(range(6))
-
-## Generic node caching code.
-
-
-class Node(object):
- """Root class for all nodes in the graph.
-
- Provides base functionality for any node in the graph, independent if it has
- children or not or if its content can be addressed through an url or needs to
- be fetched as part of another node.
-
- self.printable_attributes is only used for self documentation and for str()
- implementation.
- """
- printable_attributes = []
-
- def __init__(self, parent, url):
- self.printable_attributes = self.printable_attributes[:]
- if url:
- self.printable_attributes.append('url')
- url = url.rstrip('/')
- if parent is not None:
- self.printable_attributes.append('parent')
- self.url = url
- self.parent = parent
-
- def __str__(self):
- return self.to_string()
-
- def __repr__(self):
- """Embeds key if present."""
- key = getattr(self, 'key', None)
- if key is not None:
- return '<%s key=%s>' % (self.__class__.__name__, key)
- cached_keys = getattr(self, 'cached_keys', None)
- if cached_keys is not None:
- return '<%s keys=%s>' % (self.__class__.__name__, cached_keys)
- return super(Node, self).__repr__()
-
- def to_string(self, maximum=100):
- out = ['%s:' % self.__class__.__name__]
- assert not 'printable_attributes' in self.printable_attributes
-
- def limit(txt):
- txt = str(txt)
- if maximum > 0:
- if len(txt) > maximum + 2:
- txt = txt[:maximum] + '...'
- return txt
-
- for k in sorted(self.printable_attributes):
- if k == 'parent':
- # Avoid infinite recursion.
- continue
- out.append(limit(' %s: %r' % (k, getattr(self, k))))
- return '\n'.join(out)
-
- def refresh(self):
- """Refreshes the data."""
- self.discard()
- return self.cache()
-
- def cache(self): # pragma: no cover
- """Caches the data."""
- raise NotImplementedError()
-
- def discard(self): # pragma: no cover
- """Discards cached data.
-
- Pretty much everything is temporary except completed Build.
- """
- raise NotImplementedError()
-
-
-class AddressableBaseDataNode(Node): # pylint: disable=W0223
- """A node that contains a dictionary of data that can be fetched with an url.
-
- The node is directly addressable. It also often can be fetched by the parent.
- """
- printable_attributes = Node.printable_attributes + ['data']
-
- def __init__(self, parent, url, data):
- super(AddressableBaseDataNode, self).__init__(parent, url)
- self._data = data
-
- @property
- def cached_data(self):
- return self._data
-
- @property
- def data(self):
- self.cache()
- return self._data
-
- def cache(self):
- if self._data is None:
- self._data = self._readall()
- return True
- return False
-
- def discard(self):
- self._data = None
-
- def read(self, suburl):
- assert self.url, self.__class__.__name__
- url = self.url
- if suburl:
- url = '%s/%s' % (self.url, suburl)
- return self.parent.read(url)
-
- def _readall(self):
- return self.read('')
-
-
-class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223
- """Automatically encodes the url."""
-
- def __init__(self, parent, url, data):
- super(AddressableDataNode, self).__init__(parent, urllib.parse.quote(url),
- data)
-
-
-class NonAddressableDataNode(Node): # pylint: disable=W0223
- """A node that cannot be addressed by an unique url.
-
- The data comes directly from the parent.
- """
-
- def __init__(self, parent, subkey):
- super(NonAddressableDataNode, self).__init__(parent, None)
- self.subkey = subkey
-
- @property
- def cached_data(self):
- if self.parent.cached_data is None:
- return None
- return self.parent.cached_data[self.subkey]
-
- @property
- def data(self):
- return self.parent.data[self.subkey]
-
- def cache(self):
- self.parent.cache()
-
- def discard(self): # pragma: no cover
- """Avoid invalid state when parent recreate the object."""
- raise AttributeError('Call parent discard() instead')
-
-
-class VirtualNodeList(Node):
- """Base class for every node that has children.
-
- Adds partial supports for keys and iterator functionality. 'key' can be a
- string or a int. Not to be used directly.
- """
- printable_attributes = Node.printable_attributes + ['keys']
-
- def __init__(self, parent, url):
- super(VirtualNodeList, self).__init__(parent, url)
- # Keeps the keys independently when ordering is needed.
- self._is_cached = False
- self._has_keys_cached = False
-
- def __contains__(self, key):
- """Enables 'if i in obj:'."""
- return key in self.keys
-
- def __iter__(self):
- """Enables 'for i in obj:'. It returns children."""
- self.cache_keys()
- for key in self.keys:
- yield self[key]
-
- def __len__(self):
- """Enables 'len(obj)' to get the number of childs."""
- return len(self.keys)
-
- def discard(self):
- """Discards data.
-
- The default behavior is to not invalidate cached keys. The only place where
- keys need to be invalidated is with Builds.
- """
- self._is_cached = False
- self._has_keys_cached = False
-
- @property
- def cached_children(self): # pragma: no cover
- """Returns an iterator over the children that are cached."""
- raise NotImplementedError()
-
- @property
- def cached_keys(self): # pragma: no cover
- raise NotImplementedError()
-
- @property
- def keys(self): # pragma: no cover
- """Returns the keys for every children."""
- raise NotImplementedError()
-
- def __getitem__(self, key): # pragma: no cover
- """Returns a child, without fetching its data.
-
- The children could be invalid since no verification is done.
- """
- raise NotImplementedError()
-
- def cache(self): # pragma: no cover
- """Cache all the children."""
- raise NotImplementedError()
-
- def cache_keys(self): # pragma: no cover
- """Cache all children's keys."""
- raise NotImplementedError()
-
-
-class NodeList(VirtualNodeList): # pylint: disable=W0223
- """Adds a cache of the keys."""
-
- def __init__(self, parent, url):
- super(NodeList, self).__init__(parent, url)
- self._keys = []
-
- @property
- def cached_keys(self):
- return self._keys
-
- @property
- def keys(self):
- self.cache_keys()
- return self._keys
-
-
-class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223
- """A node that contains children but retrieves all its data from its parent.
-
- I.e. there's no url to get directly this data.
- """
- # Child class object for children of this instance. For example, BuildSteps
- # has BuildStep children.
- _child_cls = None
-
- def __init__(self, parent, subkey):
- super(NonAddressableNodeList, self).__init__(parent, None)
- self.subkey = subkey
- assert (not isinstance(self._child_cls, NonAddressableDataNode) and
- issubclass(self._child_cls, NonAddressableDataNode)), (
- self._child_cls.__name__)
-
- @property
- def cached_children(self):
- if self.parent.cached_data is not None:
- for i in range(len(self.parent.cached_data[self.subkey])):
- yield self[i]
-
- @property
- def cached_data(self):
- if self.parent.cached_data is None:
- return None
- return self.parent.data.get(self.subkey, None)
-
- @property
- def cached_keys(self):
- if self.parent.cached_data is None:
- return None
- return list(range(len(self.parent.data.get(self.subkey, []))))
-
- @property
- def data(self):
- return self.parent.data[self.subkey]
-
- def cache(self):
- self.parent.cache()
-
- def cache_keys(self):
- self.parent.cache()
-
- def discard(self): # pragma: no cover
- """Do not call.
-
- Avoid infinite recursion by having the caller calls the parent's
- discard() explicitely.
- """
- raise AttributeError('Call parent discard() instead')
-
- def __iter__(self):
- """Enables 'for i in obj:'. It returns children."""
- if self.data:
- for i in range(len(self.data)):
- yield self[i]
-
- def __getitem__(self, key):
- """Doesn't cache the value, it's not needed.
-
- TODO(maruel): Cache?
- """
- if isinstance(key, int) and key < 0:
- key = len(self.data) + key
- # pylint: disable=E1102
- return self._child_cls(self, key)
-
-
-class AddressableNodeList(NodeList):
- """A node that has children that can be addressed with an url."""
-
- # Child class object for children of this instance. For example, Builders has
- # Builder children and Builds has Build children.
- _child_cls = None
-
- def __init__(self, parent, url):
- super(AddressableNodeList, self).__init__(parent, url)
- self._cache = {}
- assert (not isinstance(self._child_cls, AddressableDataNode) and
- issubclass(self._child_cls, AddressableDataNode)), (
- self._child_cls.__name__)
-
- @property
- def cached_children(self):
- for item in self._cache.values():
- if item.cached_data is not None:
- yield item
-
- @property
- def cached_keys(self):
- return list(self._cache.keys())
-
- def __getitem__(self, key):
- """Enables 'obj[i]'."""
- if self._has_keys_cached and not key in self._keys:
- raise KeyError(key)
-
- if not key in self._cache:
- # Create an empty object.
- self._create_obj(key, None)
- return self._cache[key]
-
- def cache(self):
- if not self._is_cached:
- data = self._readall()
- for key in sorted(data):
- self._create_obj(key, data[key])
- self._is_cached = True
- self._has_keys_cached = True
-
- def cache_partial(self, children):
- """Caches a partial number of children.
-
- This method is more efficient since it does a single request for all the
- children instead of one request per children.
-
- It only grab objects not already cached.
- """
- # pylint: disable=W0212
- if not self._is_cached:
- to_fetch = [
- child for child in children
- if not (child in self._cache and self._cache[child].cached_data)
- ]
- if to_fetch:
- # Similar to cache(). The only reason to sort is to simplify testing.
- params = '&'.join(
- 'select=%s' % urllib.parse.quote(str(v)) for v in sorted(to_fetch))
- data = self.read('?' + params)
- for key in sorted(data):
- self._create_obj(key, data[key])
-
- def cache_keys(self):
- """Implement to speed up enumeration. Defaults to call cache()."""
- if not self._has_keys_cached:
- self.cache()
- assert self._has_keys_cached
-
- def discard(self):
- """Discards temporary children."""
- super(AddressableNodeList, self).discard()
- for v in self._cache.values():
- v.discard()
-
- def read(self, suburl):
- assert self.url, self.__class__.__name__
- url = self.url
- if suburl:
- url = '%s/%s' % (self.url, suburl)
- return self.parent.read(url)
-
- def _create_obj(self, key, data):
- """Creates an object of type self._child_cls."""
- # pylint: disable=E1102
- obj = self._child_cls(self, key, data)
- # obj.key and key may be different.
- # No need to overide cached data with None.
- if data is not None or obj.key not in self._cache:
- self._cache[obj.key] = obj
- if obj.key not in self._keys:
- self._keys.append(obj.key)
-
- def _readall(self):
- return self.read('')
-
-
-class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223
- """A node that shows a subset of children that comes from another structure.
-
- The node is not addressable.
-
- E.g. the keys are retrieved from parent but the actual data comes from
- virtual_parent.
- """
-
- def __init__(self, parent, virtual_parent, subkey):
- super(SubViewNodeList, self).__init__(parent, None)
- self.subkey = subkey
- self.virtual_parent = virtual_parent
- assert isinstance(self.parent, AddressableDataNode)
- assert isinstance(self.virtual_parent, NodeList)
-
- @property
- def cached_children(self):
- if self.parent.cached_data is not None:
- for item in self.keys:
- if item in self.virtual_parent.keys:
- child = self[item]
- if child.cached_data is not None:
- yield child
-
- @property
- def cached_keys(self):
- return (self.parent.cached_data or {}).get(self.subkey, [])
-
- @property
- def keys(self):
- self.cache_keys()
- return self.parent.data.get(self.subkey, [])
-
- def cache(self):
- """Batch request for each child in a single read request."""
- if not self._is_cached:
- self.virtual_parent.cache_partial(self.keys)
- self._is_cached = True
-
- def cache_keys(self):
- if not self._has_keys_cached:
- self.parent.cache()
- self._has_keys_cached = True
-
- def discard(self):
- if self.parent.cached_data is not None:
- for child in self.virtual_parent.cached_children:
- if child.key in self.keys:
- child.discard()
- self.parent.discard()
- super(SubViewNodeList, self).discard()
-
- def __getitem__(self, key):
- """Makes sure the key is in our key but grab it from the virtual parent."""
- return self.virtual_parent[key]
-
- def __iter__(self):
- self.cache()
- return super(SubViewNodeList, self).__iter__()
-
-
-# Buildbot-specific code
-
-
-class Slave(AddressableDataNode):
- """Buildbot slave class."""
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'name',
- 'key',
- 'connected',
- 'version',
- ]
-
- def __init__(self, parent, name, data):
- super(Slave, self).__init__(parent, name, data)
- self.name = name
- self.key = self.name
- # TODO(maruel): Add SlaveBuilders and a 'builders' property.
- # TODO(maruel): Add a 'running_builds' property.
-
- @property
- def connected(self):
- return self.data.get('connected', False)
-
- @property
- def version(self):
- return self.data.get('version')
-
-
-class Slaves(AddressableNodeList):
- """Buildbot slaves."""
- _child_cls = Slave
- printable_attributes = AddressableNodeList.printable_attributes + ['names']
-
- def __init__(self, parent):
- super(Slaves, self).__init__(parent, 'slaves')
-
- @property
- def names(self):
- return self.keys
-
-
-class BuilderSlaves(SubViewNodeList):
- """Similar to Slaves but only list slaves connected to a specific builder."""
- printable_attributes = SubViewNodeList.printable_attributes + ['names']
-
- def __init__(self, parent):
- super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves,
- 'slaves')
-
- @property
- def names(self):
- return self.keys
-
-
-class BuildStep(NonAddressableDataNode):
- """Class for a buildbot build step."""
- printable_attributes = NonAddressableDataNode.printable_attributes + [
- 'name',
- 'number',
- 'start_time',
- 'end_time',
- 'duration',
- 'is_started',
- 'is_finished',
- 'is_running',
- 'result',
- 'simplified_result',
- ]
-
- def __init__(self, parent, number):
- """Pre-loaded, since the data is retrieved via the Build object."""
- assert isinstance(number, int)
- super(BuildStep, self).__init__(parent, number)
- self.number = number
-
- @property
- def start_time(self):
- if self.data.get('times'):
- return int(round(self.data['times'][0]))
-
- @property
- def end_time(self):
- times = self.data.get('times')
- if times and len(times) == 2 and times[1]:
- return int(round(times[1]))
-
- @property
- def duration(self):
- if self.start_time:
- return (self.end_time or int(round(time.time()))) - self.start_time
-
- @property
- def name(self):
- return self.data['name']
-
- @property
- def is_started(self):
- return self.data.get('isStarted', False)
-
- @property
- def is_finished(self):
- return self.data.get('isFinished', False)
-
- @property
- def is_running(self):
- return self.is_started and not self.is_finished
-
- @property
- def result(self):
- result = self.data.get('results')
- if result is None:
- # results may be 0, in that case with filter=1, the value won't be
- # present.
- if self.data.get('isFinished'):
- result = self.data.get('results', 0)
- while isinstance(result, list):
- result = result[0]
- return result
-
- @property
- def simplified_result(self):
- """Returns a simplified 3 state value, True, False or None."""
- result = self.result
- if result in (SUCCESS, WARNINGS):
- return True
- elif result in (FAILURE, EXCEPTION, RETRY):
- return False
- assert result in (None, SKIPPED), (result, self.data)
- return None
-
-
-class BuildSteps(NonAddressableNodeList):
- """Duplicates keys to support lookup by both step number and step name."""
- printable_attributes = NonAddressableNodeList.printable_attributes + [
- 'failed',
- ]
- _child_cls = BuildStep
-
- def __init__(self, parent):
- """Pre-loaded, since the data is retrieved via the Build object."""
- super(BuildSteps, self).__init__(parent, 'steps')
-
- @property
- def keys(self):
- """Returns the steps name in order."""
- return [i['name'] for i in self.data or []]
-
- @property
- def failed(self):
- """Shortcuts that lists the step names of steps that failed."""
- return [step.name for step in self if step.simplified_result is False]
-
- def __getitem__(self, key):
- """Accept step name in addition to index number."""
- if isinstance(key, str):
- # It's a string, try to find the corresponding index.
- for i, step in enumerate(self.data):
- if step['name'] == key:
- key = i
- break
- else:
- raise KeyError(key)
- return super(BuildSteps, self).__getitem__(key)
-
-
-class Build(AddressableDataNode):
- """Buildbot build info."""
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'key',
- 'number',
- 'steps',
- 'blame',
- 'reason',
- 'revision',
- 'result',
- 'simplified_result',
- 'start_time',
- 'end_time',
- 'duration',
- 'slave',
- 'properties',
- 'completed',
- ]
-
- def __init__(self, parent, key, data):
- super(Build, self).__init__(parent, str(key), data)
- self.number = int(key)
- self.key = self.number
- self.steps = BuildSteps(self)
-
- @property
- def blame(self):
- return self.data.get('blame', [])
-
- @property
- def builder(self):
- """Returns the Builder object.
-
- Goes up the hierarchy to find the Buildbot.builders[builder] instance.
- """
- return self.parent.parent.parent.parent.builders[self.data['builderName']]
-
- @property
- def start_time(self):
- if self.data.get('times'):
- return int(round(self.data['times'][0]))
-
- @property
- def end_time(self):
- times = self.data.get('times')
- if times and len(times) == 2 and times[1]:
- return int(round(times[1]))
-
- @property
- def duration(self):
- if self.start_time:
- return (self.end_time or int(round(time.time()))) - self.start_time
-
- @property
- def eta(self):
- return self.data.get('eta', 0)
-
- @property
- def completed(self):
- return self.data.get('currentStep') is None
-
- @property
- def properties(self):
- return self.data.get('properties', [])
-
- @property
- def reason(self):
- return self.data.get('reason')
-
- @property
- def result(self):
- result = self.data.get('results')
- while isinstance(result, list):
- result = result[0]
- if result is None and self.steps:
- # results may be 0, in that case with filter=1, the value won't be
- # present.
- result = self.steps[-1].result
- return result
-
- @property
- def revision(self):
- return self.data.get('sourceStamp', {}).get('revision')
-
- @property
- def simplified_result(self):
- """Returns a simplified 3 state value, True, False or None."""
- result = self.result
- if result in (SUCCESS, WARNINGS, SKIPPED):
- return True
- elif result in (FAILURE, EXCEPTION, RETRY):
- return False
- assert result is None, (result, self.data)
- return None
-
- @property
- def slave(self):
- """Returns the Slave object.
-
- Goes up the hierarchy to find the Buildbot.slaves[slave] instance.
- """
- return self.parent.parent.parent.parent.slaves[self.data['slave']]
-
- def discard(self):
- """Completed Build isn't discarded."""
- if self._data and self.result is None:
- assert not self.steps or not self.steps[-1].data.get('isFinished')
- self._data = None
-
-
-class CurrentBuilds(SubViewNodeList):
- """Lists of the current builds."""
-
- def __init__(self, parent):
- super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds')
-
-
-class PendingBuilds(AddressableDataNode):
- """List of the pending builds."""
-
- def __init__(self, parent):
- super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None)
-
-
-class Builds(AddressableNodeList):
- """Supports iteration.
-
- Recommends using .cache() to speed up if a significant number of builds are
- iterated over.
- """
- _child_cls = Build
-
- def __init__(self, parent):
- super(Builds, self).__init__(parent, 'builds')
-
- def __getitem__(self, key):
- """Support for negative reference and enable retrieving non-cached builds.
-
- e.g. -1 is the last build, -2 is the previous build before the last one.
- """
- key = int(key)
- if key < 0:
- # Convert negative to positive build number.
- self.cache_keys()
- # Since the negative value can be outside of the cache keys range, use the
- # highest key value and calculate from it.
- key = max(self._keys) + key + 1
-
- if key not in self._cache:
- # Create an empty object.
- self._create_obj(key, None)
- return self._cache[key]
-
- def __iter__(self):
- """Returns cached Build objects in reversed order.
-
- The most recent build is returned first and then in reverse chronological
- order, up to the oldest cached build by the server. Older builds can be
- accessed but will trigger significantly more I/O so they are not included by
- default in the iteration.
-
- To access the older builds, use self.iterall() instead.
- """
- self.cache()
- return reversed(list(self._cache.values()))
-
- def iterall(self):
- """Returns Build objects in decreasing order unbounded up to build 0.
-
- The most recent build is returned first and then in reverse chronological
- order. Older builds can be accessed and will trigger significantly more I/O
- so use this carefully.
- """
- # Only cache keys here.
- self.cache_keys()
- if self._keys:
- for i in range(max(self._keys), -1, -1):
- yield self[i]
-
- def cache_keys(self):
- """Grabs the keys (build numbers) from the builder."""
- if not self._has_keys_cached:
- for i in self.parent.data.get('cachedBuilds', []):
- i = int(i)
- self._cache.setdefault(i, Build(self, i, None))
- if i not in self._keys:
- self._keys.append(i)
- self._has_keys_cached = True
-
- def discard(self):
- super(Builds, self).discard()
- # Can't keep keys.
- self._has_keys_cached = False
-
- def _readall(self):
- return self.read('_all')
-
-
-class Builder(AddressableDataNode):
- """Builder status."""
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'name',
- 'key',
- 'builds',
- 'slaves',
- 'pending_builds',
- 'current_builds',
- ]
-
- def __init__(self, parent, name, data):
- super(Builder, self).__init__(parent, name, data)
- self.name = name
- self.key = name
- self.builds = Builds(self)
- self.slaves = BuilderSlaves(self)
- self.current_builds = CurrentBuilds(self)
- self.pending_builds = PendingBuilds(self)
-
- def discard(self):
- super(Builder, self).discard()
- self.builds.discard()
- self.slaves.discard()
- self.current_builds.discard()
-
-
-class Builders(AddressableNodeList):
- """Root list of builders."""
- _child_cls = Builder
-
- def __init__(self, parent):
- super(Builders, self).__init__(parent, 'builders')
-
-
-class Buildbot(AddressableBaseDataNode):
- """This object should be recreated on a master restart as it caches data."""
- # Throttle fetches to not kill the server.
- auto_throttle = None
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'slaves',
- 'builders',
- 'last_fetch',
- ]
-
- def __init__(self, url):
- super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None)
- self._builders = Builders(self)
- self._slaves = Slaves(self)
- self.last_fetch = None
-
- @property
- def builders(self):
- return self._builders
-
- @property
- def slaves(self):
- return self._slaves
-
- def discard(self):
- """Discards information about Builders and Slaves."""
- super(Buildbot, self).discard()
- self._builders.discard()
- self._slaves.discard()
-
- def read(self, suburl):
- if self.auto_throttle:
- if self.last_fetch:
- delta = datetime.datetime.utcnow() - self.last_fetch
- remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta)
- if remaining > datetime.timedelta(seconds=0):
- logging.debug('Sleeping for %ss', remaining)
- time.sleep(remaining.seconds)
- self.last_fetch = datetime.datetime.utcnow()
- url = '%s/%s' % (self.url, suburl)
- if '?' in url:
- url += '&filter=1'
- else:
- url += '?filter=1'
- logging.info('read(%s)', suburl)
- channel = urllib.request.urlopen(url)
- data = channel.read()
- try:
- return json.loads(data)
- except ValueError:
- if channel.getcode() >= 400:
- # Convert it into an HTTPError for easier processing.
- raise urllib.error.HTTPError(url, channel.getcode(),
- '%s:\n%s' % (url, data), channel.headers,
- None)
- raise
-
- def _readall(self):
- return self.read('project')
-
-
-# Controller code
-
-
-def usage(more):
-
- def hook(fn):
- fn.func_usage_more = more
- return fn
-
- return hook
-
-
-def need_buildbot(fn):
- """Post-parse args to create a buildbot object."""
-
- @functools.wraps(fn)
- def hook(parser, args, *extra_args, **kwargs):
- old_parse_args = parser.parse_args
-
- def new_parse_args(args):
- options, args = old_parse_args(args)
- if len(args) < 1:
- parser.error('Need to pass the root url of the buildbot')
- url = args.pop(0)
- if not url.startswith('http'):
- url = 'http://' + url
- buildbot = Buildbot(url)
- buildbot.auto_throttle = options.throttle
- return options, args, buildbot
-
- parser.parse_args = new_parse_args
- # Call the original function with the modified parser.
- return fn(parser, args, *extra_args, **kwargs)
-
- hook.func_usage_more = '[options] <url>'
- return hook
-
-
-@need_buildbot
-def CMDpending(parser, args):
- """Lists pending jobs."""
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.builders:
- options.builders = buildbot.builders.keys
- for builder in options.builders:
- builder = buildbot.builders[builder]
- pending_builds = builder.data.get('pendingBuilds', 0)
- if not pending_builds:
- continue
- print('Builder %s: %d' % (builder.name, pending_builds))
- if not options.quiet:
- for pending in builder.pending_builds.data:
- if 'revision' in pending['source']:
- print(' revision: %s' % pending['source']['revision'])
- for change in pending['source']['changes']:
- print(' change:')
- print(' comment: %r' % change['comments'][:50])
- print(' who: %s' % change['who'])
- return 0
-
-
-@usage('[options] <url> [commands] ...')
-@need_buildbot
-def CMDrun(parser, args):
- """Runs commands passed as parameters.
-
- When passing commands on the command line, each command will be run as if it
- was on its own line.
- """
- parser.add_option('-f', '--file', help='Read script from file')
- parser.add_option(
- '-i', dest='use_stdin', action='store_true', help='Read script on stdin')
- # Variable 'buildbot' is not used directly.
- # pylint: disable=W0612
- options, args, _ = parser.parse_args(args)
- if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1:
- parser.error('Need to pass only one of: <commands>, -f <file> or -i')
- if options.use_stdin:
- cmds = sys.stdin.read()
- elif options.file:
- cmds = open(options.file).read()
- else:
- cmds = '\n'.join(args)
- compiled = compile(cmds, '<cmd line>', 'exec')
- # pylint: disable=eval-used
- eval(compiled, globals(), locals())
- return 0
-
-
-@need_buildbot
-def CMDinteractive(parser, args):
- """Runs an interactive shell to run queries."""
- _, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- prompt = ('Buildbot interactive console for "%s".\n'
- "Hint: Start with typing: 'buildbot.printable_attributes' or "
- "'print str(buildbot)' to explore.") % buildbot.url[:-len('/json')]
- local_vars = {'buildbot': buildbot, 'b': buildbot}
- code.interact(prompt, None, local_vars)
-
-
-@need_buildbot
-def CMDidle(parser, args):
- """Lists idle slaves."""
- return find_idle_busy_slaves(parser, args, True)
-
-
-@need_buildbot
-def CMDbusy(parser, args):
- """Lists idle slaves."""
- return find_idle_busy_slaves(parser, args, False)
-
-
-@need_buildbot
-def CMDdisconnected(parser, args):
- """Lists disconnected slaves."""
- _, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- for slave in buildbot.slaves:
- if not slave.connected:
- print(slave.name)
- return 0
-
-
-def find_idle_busy_slaves(parser, args, show_idle):
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '-s',
- '--slave',
- dest='slaves',
- action='append',
- default=[],
- help='Slaves to filter on')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.builders:
- options.builders = buildbot.builders.keys
- for builder in options.builders:
- builder = buildbot.builders[builder]
- if options.slaves:
- # Only the subset of slaves connected to the builder.
- slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
- if not slaves:
- continue
- else:
- slaves = builder.slaves.names
- busy_slaves = [build.slave.name for build in builder.current_builds]
- if show_idle:
- slaves = natsorted(set(slaves) - set(busy_slaves))
- else:
- slaves = natsorted(set(slaves) & set(busy_slaves))
- if options.quiet:
- for slave in slaves:
- print(slave)
- else:
- if slaves:
- print('Builder %s: %s' % (builder.name, ', '.join(slaves)))
- return 0
-
-
-def last_failure(buildbot,
- builders=None,
- slaves=None,
- steps=None,
- no_cache=False):
- """Returns Build object with last failure with the specific filters."""
- builders = builders or buildbot.builders.keys
- for builder in builders:
- builder = buildbot.builders[builder]
- if slaves:
- # Only the subset of slaves connected to the builder.
- builder_slaves = list(set(slaves).intersection(set(builder.slaves.names)))
- if not builder_slaves:
- continue
- else:
- builder_slaves = builder.slaves.names
-
- if not no_cache and len(builder.slaves) > 2:
- # Unless you just want the last few builds, it's often faster to
- # fetch the whole thing at once, at the cost of a small hickup on
- # the buildbot.
- # TODO(maruel): Cache only N last builds or all builds since
- # datetime.
- builder.builds.cache()
-
- found = []
- for build in builder.builds:
- if build.slave.name not in builder_slaves or build.slave.name in found:
- continue
- # Only add the slave for the first completed build but still look for
- # incomplete builds.
- if build.completed:
- found.append(build.slave.name)
-
- if steps:
- if any(build.steps[step].simplified_result is False for step in steps):
- yield build
- elif build.simplified_result is False:
- yield build
-
- if len(found) == len(builder_slaves):
- # Found all the slaves, quit.
- break
-
-
-@need_buildbot
-def CMDlast_failure(parser, args):
- """Lists all slaves that failed on that step on their last build.
-
- Examples:
- To find all slaves where their last build was a compile failure,
- run with --step compile
- """
- parser.add_option(
- '-S',
- '--step',
- dest='steps',
- action='append',
- default=[],
- help='List all slaves that failed on that step on their last build')
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '-s',
- '--slave',
- dest='slaves',
- action='append',
- default=[],
- help='Slaves to filter on')
- parser.add_option(
- '-n',
- '--no_cache',
- action='store_true',
- help="Don't load all builds at once")
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- print_builders = not options.quiet and len(options.builders) != 1
- last_builder = None
- for build in last_failure(
- buildbot,
- builders=options.builders,
- slaves=options.slaves,
- steps=options.steps,
- no_cache=options.no_cache):
-
- if print_builders and last_builder != build.builder:
- print(build.builder.name)
- last_builder = build.builder
-
- if options.quiet:
- if options.slaves:
- print('%s: %s' % (build.builder.name, build.slave.name))
- else:
- print(build.slave.name)
- else:
- out = '%d on %s: blame:%s' % (build.number, build.slave.name, ', '.join(
- build.blame))
- if print_builders:
- out = ' ' + out
- print(out)
-
- if len(options.steps) != 1:
- for step in build.steps:
- if step.simplified_result is False:
- # Assume the first line is the text name anyway.
- summary = ', '.join(step.data['text'][1:])[:40]
- out = ' %s: "%s"' % (step.data['name'], summary)
- if print_builders:
- out = ' ' + out
- print(out)
- return 0
-
-
-@need_buildbot
-def CMDcurrent(parser, args):
- """Lists current jobs."""
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '--blame', action='store_true', help='Only print the blame list')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.builders:
- options.builders = buildbot.builders.keys
-
- if options.blame:
- blame = set()
- for builder in options.builders:
- for build in buildbot.builders[builder].current_builds:
- if build.blame:
- for blamed in build.blame:
- blame.add(blamed)
- print('\n'.join(blame))
- return 0
-
- for builder in options.builders:
- builder = buildbot.builders[builder]
- if not options.quiet and builder.current_builds:
- print(builder.name)
- for build in builder.current_builds:
- if options.quiet:
- print(build.slave.name)
- else:
- out = '%4d: slave=%10s' % (build.number, build.slave.name)
- out += ' duration=%5d' % (build.duration or 0)
- if build.eta:
- out += ' eta=%5.0f' % build.eta
- else:
- out += ' '
- if build.blame:
- out += ' blame=' + ', '.join(build.blame)
- print(out)
-
- return 0
-
-
-@need_buildbot
-def CMDbuilds(parser, args):
- """Lists all builds.
-
- Examples:
- To find all builds on a single slave, run with -b bar -s foo.
- """
- parser.add_option(
- '-r', '--result', type='int', help='Build result to filter on')
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '-s',
- '--slave',
- dest='slaves',
- action='append',
- default=[],
- help='Slaves to filter on')
- parser.add_option(
- '-n',
- '--no_cache',
- action='store_true',
- help="Don't load all builds at once")
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- builders = options.builders or buildbot.builders.keys
- for builder in builders:
- builder = buildbot.builders[builder]
- for build in builder.builds:
- if not options.slaves or build.slave.name in options.slaves:
- if options.quiet:
- out = ''
- if options.builders:
- out += '%s/' % builder.name
- if len(options.slaves) != 1:
- out += '%s/' % build.slave.name
- out += '%d revision:%s result:%s blame:%s' % (
- build.number, build.revision, build.result, ','.join(build.blame))
- print(out)
- else:
- print(build)
- return 0
-
-
-@need_buildbot
-def CMDcount(parser, args):
- """Count the number of builds that occured during a specific period."""
- parser.add_option(
- '-o', '--over', type='int', help='Number of seconds to look for')
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.over:
- parser.error(
- 'Specify the number of seconds, e.g. --over 86400 for the last 24 '
- 'hours')
- builders = options.builders or buildbot.builders.keys
- counts = {}
- since = time.time() - options.over
- for builder in builders:
- builder = buildbot.builders[builder]
- counts[builder.name] = 0
- if not options.quiet:
- print(builder.name)
- for build in builder.builds.iterall():
- try:
- start_time = build.start_time
- except urllib.error.HTTPError:
- # The build was probably trimmed.
- print(
- 'Failed to fetch build %s/%d' % (builder.name, build.number),
- file=sys.stderr)
- continue
- if start_time >= since:
- counts[builder.name] += 1
- else:
- break
- if not options.quiet:
- print('.. %d' % counts[builder.name])
-
- align_name = max(len(b) for b in counts)
- align_number = max(len(str(c)) for c in counts.values())
- for builder in sorted(counts):
- print('%*s: %*d' % (align_name, builder, align_number, counts[builder]))
- print('Total: %d' % sum(counts.values()))
- return 0
-
-
-def gen_parser():
- """Returns an OptionParser instance with default options.
-
- It should be then processed with gen_usage() before being used.
- """
- parser = optparse.OptionParser(version=__version__)
- # Remove description formatting
- parser.format_description = lambda x: parser.description
- # Add common parsing.
- old_parser_args = parser.parse_args
-
- def Parse(*args, **kwargs):
- options, args = old_parser_args(*args, **kwargs)
- if options.verbose >= 2:
- logging.basicConfig(level=logging.DEBUG)
- elif options.verbose:
- logging.basicConfig(level=logging.INFO)
- else:
- logging.basicConfig(level=logging.WARNING)
- return options, args
-
- parser.parse_args = Parse
-
- parser.add_option(
- '-v',
- '--verbose',
- action='count',
- help='Use multiple times to increase logging leve')
- parser.add_option(
- '-q',
- '--quiet',
- action='store_true',
- help='Reduces the output to be parsed by scripts, independent of -v')
- parser.add_option(
- '--throttle',
- type='float',
- help='Minimum delay to sleep between requests')
- return parser
-
-
-# Generic subcommand handling code
-
-
-def Command(name):
- return getattr(sys.modules[__name__], 'CMD' + name, None)
-
-
-@usage('<command>')
-def CMDhelp(parser, args):
- """Print list of commands or use 'help <command>'."""
- _, args = parser.parse_args(args)
- if len(args) == 1:
- return main(args + ['--help'])
- parser.print_help()
- return 0
-
-
-def gen_usage(parser, command):
- """Modifies an OptionParser object with the command's documentation.
-
- The documentation is taken from the function's docstring.
- """
- obj = Command(command)
- more = getattr(obj, 'func_usage_more')
- # OptParser.description prefer nicely non-formatted strings.
- parser.description = obj.__doc__ + '\n'
- parser.set_usage('usage: %%prog %s %s' % (command, more))
-
-
-def main(args=None):
- # Do it late so all commands are listed.
- # pylint: disable=E1101
- CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join(
- ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0])
- for fn in dir(sys.modules[__name__])
- if fn.startswith('CMD'))
-
- parser = gen_parser()
- if args is None:
- args = sys.argv[1:]
- if args:
- command = Command(args[0])
- if command:
- # "fix" the usage and the description now that we know the subcommand.
- gen_usage(parser, args[0])
- return command(parser, args[1:])
-
- # Not a known command. Default to help.
- gen_usage(parser, 'help')
- return CMDhelp(parser, args)
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py
index e0c54782..b600c6aa 100644
--- a/cros_utils/buildbot_utils.py
+++ b/cros_utils/buildbot_utils.py
@@ -50,8 +50,8 @@ def PeekTrybotImage(chromeos_root, buildbucket_id):
and url looks like:
gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
"""
- command = (
- 'cros buildresult --report json --buildbucket-id %s' % buildbucket_id)
+ command = ('cros buildresult --report json --buildbucket-id %s' %
+ buildbucket_id)
rc, out, _ = RunCommandInPath(chromeos_root, command)
# Current implementation of cros buildresult returns fail when a job is still
@@ -194,9 +194,9 @@ def GetTrybotImage(chromeos_root,
image = ''
if not image:
- logger.GetLogger().LogError(
- 'Trybot job (buildbucket id: %s) failed with'
- 'status %s; no trybot image generated. ' % (buildbucket_id, status))
+ logger.GetLogger().LogError('Trybot job (buildbucket id: %s) failed with'
+ 'status %s; no trybot image generated. ' %
+ (buildbucket_id, status))
else:
# Convert full gs path to what crosperf expects. For example, convert
# gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
@@ -226,13 +226,13 @@ def WaitForImage(chromeos_root, build):
while elapsed_time < TIME_OUT:
if DoesImageExist(chromeos_root, build):
return
- logger.GetLogger().LogOutput(
- 'Image %s not ready, waiting for 10 minutes' % build)
+ logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
+ build)
time.sleep(SLEEP_TIME)
elapsed_time += SLEEP_TIME
- logger.GetLogger().LogOutput(
- 'Image %s not found, waited for %d hours' % (build, (TIME_OUT / 3600)))
+ logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
+ (build, (TIME_OUT / 3600)))
raise BuildbotTimeout('Timeout while waiting for image %s' % build)
@@ -253,7 +253,7 @@ def GetLatestImage(chromeos_root, path):
candidates.sort(reverse=True)
for c in candidates:
build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3])
- # Blacklist "R79-12384.0.0" image released by mistake.
+ # Denylist "R79-12384.0.0" image released by mistake.
# TODO(crbug.com/992242): Remove the filter by 2019-09-05.
if c == [79, 12384, 0, 0]:
continue
diff --git a/cros_utils/buildbot_utils_unittest.py b/cros_utils/buildbot_utils_unittest.py
index afbdfaef..c615c95f 100755
--- a/cros_utils/buildbot_utils_unittest.py
+++ b/cros_utils/buildbot_utils_unittest.py
@@ -21,18 +21,6 @@ from cros_utils import command_executer
class TrybotTest(unittest.TestCase):
"""Test for CommandExecuter class."""
- old_tryjob_out = (
- 'Verifying patches...\n'
- 'Submitting tryjob...\n'
- 'Successfully sent PUT request to [buildbucket_bucket:master.chromiumos.t'
- 'ryserver] with [config:success-build] [buildbucket_id:895272114382368817'
- '6].\n'
- 'Tryjob submitted!\n'
- 'To view your tryjobs, visit:\n'
- ' http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbuck'
- 'etId=8952721143823688176\n'
- ' https://uberchromegw.corp.google.com/i/chromiumos.tryserver/waterfall?'
- 'committer=laszio@chromium.org&builder=etc\n')
tryjob_out = (
'[{"buildbucket_id": "8952721143823688176", "build_config": '
'"cave-llvm-toolchain-tryjob", "url": '
diff --git a/cros_utils/command_executer.py b/cros_utils/command_executer.py
index b9acbe31..aeedf3ea 100755
--- a/cros_utils/command_executer.py
+++ b/cros_utils/command_executer.py
@@ -229,9 +229,11 @@ class CommandExecuter(object):
kwargs['return_output'] = True
return self.RunCommandGeneric(*args, **kwargs)
- def RemoteAccessInitCommand(self, chromeos_root, machine):
+ def RemoteAccessInitCommand(self, chromeos_root, machine, port=None):
command = ''
command += '\nset -- --remote=' + machine
+ if port:
+ command += ' --ssh_port=' + port
command += '\n. ' + chromeos_root + '/src/scripts/common.sh'
command += '\n. ' + chromeos_root + '/src/scripts/remote_access.sh'
command += '\nTMP=$(mktemp -d)'
@@ -285,12 +287,16 @@ class CommandExecuter(object):
sys.exit(1)
chromeos_root = os.path.expanduser(chromeos_root)
+ port = None
+ if ':' in machine:
+ machine, port = machine.split(':')
# Write all commands to a file.
command_file = self.WriteToTempShFile(cmd)
retval = self.CopyFiles(
command_file,
command_file,
dest_machine=machine,
+ dest_port=port,
command_terminator=command_terminator,
chromeos_root=chromeos_root,
dest_cros=True,
@@ -302,7 +308,7 @@ class CommandExecuter(object):
' Is the machine up?')
return (retval, '', '')
- command = self.RemoteAccessInitCommand(chromeos_root, machine)
+ command = self.RemoteAccessInitCommand(chromeos_root, machine, port)
command += '\nremote_sh bash %s' % command_file
command += '\nl_retval=$?; echo "$REMOTE_OUT"; exit $l_retval'
retval = self.RunCommandGeneric(
@@ -313,8 +319,8 @@ class CommandExecuter(object):
terminated_timeout=terminated_timeout,
print_to_console=print_to_console)
if return_output:
- connect_signature = (
- 'Initiating first contact with remote host\n' + 'Connection OK\n')
+ connect_signature = ('Initiating first contact with remote host\n' +
+ 'Connection OK\n')
connect_signature_re = re.compile(connect_signature)
modded_retval = list(retval)
modded_retval[1] = connect_signature_re.sub('', retval[1])
@@ -381,7 +387,7 @@ class CommandExecuter(object):
command_file = f.name
os.chmod(command_file, 0o777)
- # if return_output is set, run a dummy command first to make sure that
+ # if return_output is set, run a test command first to make sure that
# the chroot already exists. We want the final returned output to skip
# the output from chroot creation steps.
if return_output:
@@ -449,7 +455,9 @@ class CommandExecuter(object):
src,
dest,
src_machine=None,
+ src_port=None,
dest_machine=None,
+ dest_port=None,
src_user=None,
dest_user=None,
recursive=True,
@@ -475,30 +483,34 @@ class CommandExecuter(object):
sys.exit(1)
if src_cros:
cros_machine = src_machine
+ cros_port = src_port
+ host_machine = dest_machine
+ host_user = dest_user
else:
cros_machine = dest_machine
-
- command = self.RemoteAccessInitCommand(chromeos_root, cros_machine)
- ssh_command = (
- 'ssh -o StrictHostKeyChecking=no' + ' -o UserKnownHostsFile=$(mktemp)'
- + ' -i $TMP_PRIVATE_KEY')
+ cros_port = dest_port
+ host_machine = src_machine
+ host_user = src_user
+
+ command = self.RemoteAccessInitCommand(chromeos_root, cros_machine,
+ cros_port)
+ ssh_command = ('ssh -o StrictHostKeyChecking=no' +
+ ' -o UserKnownHostsFile=$(mktemp)' +
+ ' -i $TMP_PRIVATE_KEY')
+ if cros_port:
+ ssh_command += ' -p %s' % cros_port
rsync_prefix = '\nrsync -r -e "%s" ' % ssh_command
if dest_cros:
- command += rsync_prefix + '%s root@%s:%s' % (src, dest_machine, dest)
- return self.RunCommand(
- command,
- machine=src_machine,
- username=src_user,
- command_terminator=command_terminator,
- print_to_console=print_to_console)
+ command += rsync_prefix + '%s root@%s:%s' % (src, cros_machine, dest)
else:
- command += rsync_prefix + 'root@%s:%s %s' % (src_machine, src, dest)
- return self.RunCommand(
- command,
- machine=dest_machine,
- username=dest_user,
- command_terminator=command_terminator,
- print_to_console=print_to_console)
+ command += rsync_prefix + 'root@%s:%s %s' % (cros_machine, src, dest)
+
+ return self.RunCommand(
+ command,
+ machine=host_machine,
+ username=host_user,
+ command_terminator=command_terminator,
+ print_to_console=print_to_console)
if dest_machine == src_machine:
command = 'rsync -a %s %s' % (src, dest)
diff --git a/cros_utils/manifest_versions.py b/cros_utils/manifest_versions.py
index 115c6046..4838de3c 100644
--- a/cros_utils/manifest_versions.py
+++ b/cros_utils/manifest_versions.py
@@ -19,6 +19,8 @@ import time
from cros_utils import command_executer
from cros_utils import logger
+MANIFEST_VERSION_MAIN_BRANCH = 'master'
+
def IsCrosVersion(version):
match = re.search(r'(\d+\.\d+\.\d+\.\d+)', version)
@@ -72,7 +74,8 @@ class ManifestVersions(object):
commands = [
'cd {0}'.format(self.clone_location), 'cd manifest-versions',
'git checkout -f $(git rev-list' +
- ' --max-count=1 --before={0} origin/master)'.format(my_time)
+ ' --max-count=1 --before={0} origin/{1})'.format(
+ my_time, MANIFEST_VERSION_MAIN_BRANCH)
]
ret = self.ce.RunCommands(commands)
if ret:
@@ -106,7 +109,7 @@ class ManifestVersions(object):
version = pp[-2] + '.' + small
commands = [
'cd {0}'.format(self.clone_location), 'cd manifest-versions',
- 'git checkout master'
+ 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH)
]
self.ce.RunCommands(commands)
return version
@@ -120,7 +123,8 @@ class ManifestVersions(object):
commands = [
'cd {0}'.format(self.clone_location), 'cd manifest-versions',
'git checkout -f $(git rev-list' +
- ' --max-count=1 --before={0} origin/master)'.format(my_time)
+ ' --max-count=1 --before={0} origin/{1})'.format(
+ my_time, MANIFEST_VERSION_MAIN_BRANCH)
]
ret = self.ce.RunCommands(commands)
if ret:
@@ -133,7 +137,7 @@ class ManifestVersions(object):
version = pp[-2] + '.' + small
commands = [
'cd {0}'.format(self.clone_location), 'cd manifest-versions',
- 'git checkout master'
+ 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH)
]
self.ce.RunCommands(commands)
return version
diff --git a/cros_utils/misc.py b/cros_utils/misc.py
index b4beb60c..93d1b3b6 100644
--- a/cros_utils/misc.py
+++ b/cros_utils/misc.py
@@ -24,12 +24,15 @@ CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/'
'cros_utils/toolchain_utils.sh')
+CROS_MAIN_BRANCH = 'cros/master'
+
def GetChromeOSVersionFromLSBVersion(lsb_version):
"""Get Chromeos version from Lsb version."""
ce = command_executer.GetCommandExecuter()
command = ('git ls-remote '
- 'https://chromium.googlesource.com/chromiumos/manifest.git')
+ 'https://chromium.googlesource.com/chromiumos/manifest.git '
+ 'refs/heads/release-R*')
ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
assert ret == 0, 'Command %s failed' % command
lower = []
@@ -444,8 +447,9 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False):
cmd1, print_to_console=True) == 0
-def ApplyGerritPatches(chromeos_root, gerrit_patch_string,
- branch='cros/master'):
+def ApplyGerritPatches(chromeos_root,
+ gerrit_patch_string,
+ branch=CROS_MAIN_BRANCH):
"""Apply gerrit patches on a chromeos tree.
Args:
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py
index b4092993..1a3fd4a7 100644
--- a/cros_utils/tabulator.py
+++ b/cros_utils/tabulator.py
@@ -67,10 +67,10 @@ from __future__ import print_function
import collections
import getpass
import math
+import statistics
import sys
-# TODO(zhizhouy): Drop numpy in the future
+# TODO(crbug.com/980719): Drop scipy in the future.
# pylint: disable=import-error
-import numpy
import scipy
from cros_utils.email_sender import EmailSender
@@ -556,7 +556,7 @@ class AmeanResult(StringMeanResult):
def _ComputeFloat(self, cell, values, baseline_values):
if self.ignore_min_max:
values = _RemoveMinMax(cell, values)
- cell.value = numpy.mean(values)
+ cell.value = statistics.mean(values)
class RawResult(Result):
@@ -610,7 +610,7 @@ class StdResult(NumericalResult):
def _ComputeFloat(self, cell, values, baseline_values):
if self.ignore_min_max:
values = _RemoveMinMax(cell, values)
- cell.value = numpy.std(values)
+ cell.value = statistics.pstdev(values)
class CoeffVarResult(NumericalResult):
@@ -623,8 +623,8 @@ class CoeffVarResult(NumericalResult):
def _ComputeFloat(self, cell, values, baseline_values):
if self.ignore_min_max:
values = _RemoveMinMax(cell, values)
- if numpy.mean(values) != 0.0:
- noise = numpy.abs(numpy.std(values) / numpy.mean(values))
+ if statistics.mean(values) != 0.0:
+ noise = abs(statistics.pstdev(values) / statistics.mean(values))
else:
noise = 0.0
cell.value = noise
@@ -731,9 +731,12 @@ class AmeanRatioResult(KeyAwareComparisonResult):
if self.ignore_min_max:
values = _RemoveMinMax(cell, values)
baseline_values = _RemoveMinMax(cell, baseline_values)
- if numpy.mean(baseline_values) != 0:
- cell.value = numpy.mean(values) / numpy.mean(baseline_values)
- elif numpy.mean(values) != 0:
+
+ baseline_mean = statistics.mean(baseline_values)
+ values_mean = statistics.mean(values)
+ if baseline_mean != 0:
+ cell.value = values_mean / baseline_mean
+ elif values_mean != 0:
cell.value = 0.00
# cell.value = 0 means the values and baseline_values have big difference
else:
@@ -1495,41 +1498,40 @@ def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE):
if __name__ == '__main__':
# Run a few small tests here.
- runs = [
- [{
- 'k1': '10',
- 'k2': '12',
- 'k5': '40',
- 'k6': '40',
- 'ms_1': '20',
- 'k7': 'FAIL',
- 'k8': 'PASS',
- 'k9': 'PASS',
- 'k10': '0'
- },
- {
- 'k1': '13',
- 'k2': '14',
- 'k3': '15',
- 'ms_1': '10',
- 'k8': 'PASS',
- 'k9': 'FAIL',
- 'k10': '0'
- }],
- [{
- 'k1': '50',
- 'k2': '51',
- 'k3': '52',
- 'k4': '53',
- 'k5': '35',
- 'k6': '45',
- 'ms_1': '200',
- 'ms_2': '20',
- 'k7': 'FAIL',
- 'k8': 'PASS',
- 'k9': 'PASS'
- }],
- ]
+ run1 = {
+ 'k1': '10',
+ 'k2': '12',
+ 'k5': '40',
+ 'k6': '40',
+ 'ms_1': '20',
+ 'k7': 'FAIL',
+ 'k8': 'PASS',
+ 'k9': 'PASS',
+ 'k10': '0'
+ }
+ run2 = {
+ 'k1': '13',
+ 'k2': '14',
+ 'k3': '15',
+ 'ms_1': '10',
+ 'k8': 'PASS',
+ 'k9': 'FAIL',
+ 'k10': '0'
+ }
+ run3 = {
+ 'k1': '50',
+ 'k2': '51',
+ 'k3': '52',
+ 'k4': '53',
+ 'k5': '35',
+ 'k6': '45',
+ 'ms_1': '200',
+ 'ms_2': '20',
+ 'k7': 'FAIL',
+ 'k8': 'PASS',
+ 'k9': 'PASS'
+ }
+ runs = [[run1, run2], [run3]]
labels = ['vanilla', 'modified']
t = GetComplexTable(runs, labels, TablePrinter.CONSOLE)
print(t)
diff --git a/cros_utils/tabulator_test.py b/cros_utils/tabulator_test.py
index 227e2d70..9dd4828e 100755
--- a/cros_utils/tabulator_test.py
+++ b/cros_utils/tabulator_test.py
@@ -33,6 +33,20 @@ class TabulatorTest(unittest.TestCase):
result.Compute(cell, table[2], table[1])
self.assertTrue(cell.value == float(table[2][0]))
+ def testStdResult(self):
+ table = ['k1', [], ['1', '2']]
+ result = tabulator.StdResult()
+ cell = tabulator.Cell()
+ result.Compute(cell, table[2], table[1])
+ self.assertTrue(cell.value == 0.5)
+
+ def testStdResultOfSampleSize1(self):
+ table = ['k1', [], ['1']]
+ result = tabulator.StdResult()
+ cell = tabulator.Cell()
+ result.Compute(cell, table[2], table[1])
+ self.assertTrue(cell.value == 0.0)
+
def testStringMean(self):
smr = tabulator.StringMeanResult()
cell = tabulator.Cell()
diff --git a/crosperf/default_remotes b/crosperf/default_remotes
index 7b59c2af..cc7f33ef 100644
--- a/crosperf/default_remotes
+++ b/crosperf/default_remotes
@@ -1,8 +1,9 @@
-elm : chromeos2-row9-rack8-host19.cros chromeos2-row9-rack8-host21.cros
-bob : chromeos2-row9-rack7-host1.cros chromeos2-row9-rack7-host3.cros
-chell : chromeos2-row9-rack8-host3.cros chromeos2-row9-rack8-host5.cros
-kefka : chromeos2-row9-rack9-host21.cros chromeos2-row9-rack8-host1.cros
-lulu : chromeos2-row9-rack8-host9.cros chromeos2-row9-rack8-host7.cros
-nautilus : chromeos2-row9-rack7-host11.cros chromeos2-row9-rack7-host9.cros
-snappy : chromeos2-row9-rack7-host5.cros chromeos2-row9-rack7-host7.cros
-veyron_minnie : chromeos2-row9-rack8-host15.cros chromeos2-row9-rack8-host17.cros
+bob : chromeos2-row10-rack9-host1.cros chromeos2-row10-rack9-host3.cros
+coral : chromeos2-row9-rack9-host9.cros chromeos2-row9-rack9-host11.cros chromeos2-row9-rack9-host13.cros
+elm : chromeos2-row10-rack9-host19.cros chromeos2-row10-rack9-host21.cros
+chell : chromeos2-row9-rack9-host1.cros chromeos2-row9-rack9-host3.cros
+kefka : chromeos2-row9-rack9-host21.cros chromeos2-row10-rack9-host13.cros
+lulu : chromeos2-row9-rack9-host5.cros chromeos2-row9-rack9-host7.cros
+nautilus : chromeos2-row10-rack9-host9.cros chromeos2-row10-rack9-host11.cros
+snappy : chromeos2-row10-rack9-host5.cros chromeos2-row10-rack9-host7.cros
+veyron_minnie : chromeos2-row10-rack9-host15.cros chromeos2-row10-rack9-host17.cros
diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py
index 21fa3ea0..8ba85a4c 100644
--- a/crosperf/experiment_runner.py
+++ b/crosperf/experiment_runner.py
@@ -35,8 +35,8 @@ def _WriteJSONReportToFile(experiment, results_dir, json_report):
compiler_string = 'llvm' if has_llvm else 'gcc'
board = experiment.labels[0].board
filename = 'report_%s_%s_%s.%s.json' % (board, json_report.date,
- json_report.time.replace(':', '.'),
- compiler_string)
+ json_report.time.replace(
+ ':', '.'), compiler_string)
fullname = os.path.join(results_dir, filename)
report_text = json_report.GetReport()
with open(fullname, 'w') as out_file:
@@ -157,13 +157,13 @@ class ExperimentRunner(object):
def _ClearCacheEntries(self, experiment):
for br in experiment.benchmark_runs:
cache = ResultsCache()
- cache.Init(
- br.label.chromeos_image, br.label.chromeos_root,
- br.benchmark.test_name, br.iteration, br.test_args, br.profiler_args,
- br.machine_manager, br.machine, br.label.board, br.cache_conditions,
- br.logger(), br.log_level, br.label, br.share_cache,
- br.benchmark.suite, br.benchmark.show_all_results,
- br.benchmark.run_local, br.benchmark.cwp_dso)
+ cache.Init(br.label.chromeos_image, br.label.chromeos_root,
+ br.benchmark.test_name, br.iteration, br.test_args,
+ br.profiler_args, br.machine_manager, br.machine,
+ br.label.board, br.cache_conditions, br.logger(), br.log_level,
+ br.label, br.share_cache, br.benchmark.suite,
+ br.benchmark.show_all_results, br.benchmark.run_local,
+ br.benchmark.cwp_dso)
cache_dir = cache.GetCacheDirForWrite()
if os.path.exists(cache_dir):
self.l.LogOutput('Removing cache dir: %s' % cache_dir)
@@ -173,7 +173,7 @@ class ExperimentRunner(object):
try:
# We should not lease machines if tests are launched via `skylab
# create-test`. This is because leasing DUT in skylab will create a
- # dummy task on the DUT and new test created will be hanging there.
+ # no-op task on the DUT and new test created will be hanging there.
# TODO(zhizhouy): Need to check whether machine is ready or not before
# assigning a test to it.
if not experiment.skylab:
@@ -246,8 +246,8 @@ class ExperimentRunner(object):
subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names))
text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
- text_report += (
- '\nResults are stored in %s.\n' % experiment.results_directory)
+ text_report += ('\nResults are stored in %s.\n' %
+ experiment.results_directory)
text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report
html_report = HTMLResultsReport.FromExperiment(experiment).GetReport()
attachment = EmailSender.Attachment('report.html', html_report)
@@ -275,8 +275,8 @@ class ExperimentRunner(object):
all_failed = True
topstats_file = os.path.join(results_directory, 'topstats.log')
- self.l.LogOutput(
- 'Storing top statistics of each benchmark run into %s.' % topstats_file)
+ self.l.LogOutput('Storing top statistics of each benchmark run into %s.' %
+ topstats_file)
with open(topstats_file, 'w') as top_fd:
for benchmark_run in experiment.benchmark_runs:
if benchmark_run.result:
@@ -322,8 +322,8 @@ class ExperimentRunner(object):
self.l.LogOutput('Storing email message body in %s.' % results_directory)
msg_file_path = os.path.join(results_directory, 'msg_body.html')
text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
- text_report += (
- '\nResults are stored in %s.\n' % experiment.results_directory)
+ text_report += ('\nResults are stored in %s.\n' %
+ experiment.results_directory)
msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report
FileUtils().WriteFile(msg_file_path, msg_body)
@@ -348,8 +348,8 @@ class MockExperimentRunner(ExperimentRunner):
super(MockExperimentRunner, self).__init__(experiment, json_report)
def _Run(self, experiment):
- self.l.LogOutput(
- "Would run the following experiment: '%s'." % experiment.name)
+ self.l.LogOutput("Would run the following experiment: '%s'." %
+ experiment.name)
def _PrintTable(self, experiment):
self.l.LogOutput('Would print the experiment table.')
diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py
index e19d4695..8c3510a9 100755
--- a/crosperf/generate_report_unittest.py
+++ b/crosperf/generate_report_unittest.py
@@ -109,9 +109,9 @@ class GenerateReportTests(unittest.TestCase):
}
results = generate_report.CutResultsInPlace(
bench_data, max_keys=0, complain_on_update=False)
- # Just reach into results assuming we know it otherwise outputs things
- # sanely. If it doesn't, testCutResultsInPlace should give an indication as
- # to what, exactly, is broken.
+ # Just reach into results assuming we know it otherwise outputs things in
+ # the expected way. If it doesn't, testCutResultsInPlace should give an
+ # indication as to what, exactly, is broken.
self.assertEqual(list(results['foo'][0][0].items()), [('retval', 0)])
self.assertEqual(list(results['bar'][0][0].items()), [('retval', 1)])
self.assertEqual(list(results['baz'][0][0].items()), [])
diff --git a/crosperf/label.py b/crosperf/label.py
index b8122613..a55d663c 100644
--- a/crosperf/label.py
+++ b/crosperf/label.py
@@ -61,9 +61,9 @@ class Label(object):
if self.image_type == 'local':
chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image)
if not chromeos_root:
- raise RuntimeError(
- "No ChromeOS root given for label '%s' and could "
- "not determine one from image path: '%s'." % (name, chromeos_image))
+ raise RuntimeError("No ChromeOS root given for label '%s' and could "
+ "not determine one from image path: '%s'." %
+ (name, chromeos_image))
else:
chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root)
if not chromeos_root:
@@ -72,17 +72,31 @@ class Label(object):
self.chromeos_root = chromeos_root
if not chrome_src:
- self.chrome_src = os.path.join(
- self.chromeos_root, '.cache/distfiles/target/chrome-src-internal')
- if not os.path.exists(self.chrome_src):
- self.chrome_src = os.path.join(self.chromeos_root,
- '.cache/distfiles/target/chrome-src')
+ # Old and new chroots may have different chrome src locations.
+ # The path also depends on the chrome build flags.
+ # Give priority to chrome-src-internal.
+ chrome_src_rel_paths = [
+ '.cache/distfiles/target/chrome-src-internal',
+ '.cache/distfiles/chrome-src-internal',
+ '.cache/distfiles/target/chrome-src',
+ '.cache/distfiles/chrome-src',
+ ]
+ for chrome_src_rel_path in chrome_src_rel_paths:
+ chrome_src_abs_path = os.path.join(self.chromeos_root,
+ chrome_src_rel_path)
+ if os.path.exists(chrome_src_abs_path):
+ chrome_src = chrome_src_abs_path
+ break
+ if not chrome_src:
+ raise RuntimeError('Can not find location of Chrome sources.\n'
+ f'Checked paths: {chrome_src_rel_paths}')
else:
- chromeos_src = misc.CanonicalizePath(chrome_src)
- if not chromeos_src:
+ chrome_src = misc.CanonicalizePath(chrome_src)
+ # Make sure the path exists.
+ if not os.path.exists(chrome_src):
raise RuntimeError("Invalid Chrome src given for label '%s': '%s'." %
(name, chrome_src))
- self.chrome_src = chromeos_src
+ self.chrome_src = chrome_src
self._SetupChecksum()
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index f7b78e39..c5c85942 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -35,6 +35,14 @@ PERF_RESULTS_FILE = 'perf-results.txt'
CACHE_KEYS_FILE = 'cache_keys.txt'
+class PidVerificationError(Exception):
+ """Error of perf PID verification in per-process mode."""
+
+
+class PerfDataReadError(Exception):
+ """Error of reading a perf.data header."""
+
+
class Result(object):
"""Class for holding the results of a single test run.
@@ -58,6 +66,7 @@ class Result(object):
self.results_file = []
self.turbostat_log_file = ''
self.cpustats_log_file = ''
+ self.cpuinfo_file = ''
self.top_log_file = ''
self.wait_time_log_file = ''
self.chrome_version = ''
@@ -125,6 +134,13 @@ class Result(object):
self.CopyFilesTo(dest_dir, self.results_file)
self.CopyFilesTo(dest_dir, self.perf_data_files)
self.CopyFilesTo(dest_dir, self.perf_report_files)
+ extra_files = []
+ if self.top_log_file:
+ extra_files.append(self.top_log_file)
+ if self.cpuinfo_file:
+ extra_files.append(self.cpuinfo_file)
+ if extra_files:
+ self.CopyFilesTo(dest_dir, extra_files)
if self.results_file or self.perf_data_files or self.perf_report_files:
self._logger.LogOutput('Results files stored in %s.' % dest_dir)
@@ -134,12 +150,12 @@ class Result(object):
# while tast runs hold output under TEST_NAME/.
# Both ensure to be unique.
result_dir_name = self.test_name if self.suite == 'tast' else 'results'
- results_dir = self.FindFilesInResultsDir(
- '-name %s' % result_dir_name).split('\n')[0]
+ results_dir = self.FindFilesInResultsDir('-name %s' %
+ result_dir_name).split('\n')[0]
if not results_dir:
- self._logger.LogOutput(
- 'WARNING: No results dir matching %r found' % result_dir_name)
+ self._logger.LogOutput('WARNING: No results dir matching %r found' %
+ result_dir_name)
return
self.CreateTarball(results_dir, tarball)
@@ -231,8 +247,8 @@ class Result(object):
command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir)
self.ce.RunCommand(command, print_to_console=False)
- command = ('./generate_test_report --no-color --csv %s' % (os.path.join(
- '/tmp', os.path.basename(self.temp_dir))))
+ command = ('./generate_test_report --no-color --csv %s' %
+ (os.path.join('/tmp', os.path.basename(self.temp_dir))))
_, out, _ = self.ce.ChrootRunCommandWOutput(
self.chromeos_root, command, print_to_console=False)
keyvals_dict = {}
@@ -349,6 +365,10 @@ class Result(object):
"""Get cpustats log path string."""
return self.FindFilesInResultsDir('-name cpustats.log').split('\n')[0]
+ def GetCpuinfoFile(self):
+ """Get cpustats log path string."""
+ return self.FindFilesInResultsDir('-name cpuinfo.log').split('\n')[0]
+
def GetTopFile(self):
"""Get cpustats log path string."""
return self.FindFilesInResultsDir('-name top.log').split('\n')[0]
@@ -378,8 +398,8 @@ class Result(object):
perf_data_file)
perf_report_file = '%s.report' % perf_data_file
if os.path.exists(perf_report_file):
- raise RuntimeError(
- 'Perf report file already exists: %s' % perf_report_file)
+ raise RuntimeError('Perf report file already exists: %s' %
+ perf_report_file)
chroot_perf_report_file = misc.GetInsideChrootPath(
self.chromeos_root, perf_report_file)
perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf')
@@ -417,8 +437,8 @@ class Result(object):
if self.log_level != 'verbose':
self._logger.LogOutput('Perf report generated successfully.')
else:
- raise RuntimeError(
- 'Perf report not generated correctly. CMD: %s' % command)
+ raise RuntimeError('Perf report not generated correctly. CMD: %s' %
+ command)
# Add a keyval to the dictionary for the events captured.
perf_report_files.append(
@@ -455,6 +475,7 @@ class Result(object):
self.perf_report_files = self.GeneratePerfReportFiles()
self.turbostat_log_file = self.GetTurbostatFile()
self.cpustats_log_file = self.GetCpustatsFile()
+ self.cpuinfo_file = self.GetCpuinfoFile()
self.top_log_file = self.GetTopFile()
self.wait_time_log_file = self.GetWaitTimeFile()
# TODO(asharif): Do something similar with perf stat.
@@ -831,6 +852,88 @@ class Result(object):
keyvals[key] = [result, unit]
return keyvals
+ def ReadPidFromPerfData(self):
+ """Read PIDs from perf.data files.
+
+ Extract PID from perf.data if "perf record" was running per process,
+ i.e. with "-p <PID>" and no "-a".
+
+ Returns:
+ pids: list of PIDs.
+
+ Raises:
+ PerfDataReadError when perf.data header reading fails.
+ """
+ cmd = ['/usr/bin/perf', 'report', '--header-only', '-i']
+ pids = []
+
+ for perf_data_path in self.perf_data_files:
+ perf_data_path_in_chroot = misc.GetInsideChrootPath(
+ self.chromeos_root, perf_data_path)
+ path_str = ' '.join(cmd + [perf_data_path_in_chroot])
+ status, output, _ = self.ce.ChrootRunCommandWOutput(
+ self.chromeos_root, path_str)
+ if status:
+ # Error of reading a perf.data profile is fatal.
+ raise PerfDataReadError(f'Failed to read perf.data profile: {path_str}')
+
+ # Pattern to search a line with "perf record" command line:
+ # # cmdline : /usr/bin/perf record -e instructions -p 123"
+ cmdline_regex = re.compile(
+ r'^\#\scmdline\s:\s+(?P<cmd>.*perf\s+record\s+.*)$')
+ # Pattern to search PID in a command line.
+ pid_regex = re.compile(r'^.*\s-p\s(?P<pid>\d+)\s*.*$')
+ for line in output.splitlines():
+ cmd_match = cmdline_regex.match(line)
+ if cmd_match:
+ # Found a perf command line.
+ cmdline = cmd_match.group('cmd')
+ # '-a' is a system-wide mode argument.
+ if '-a' not in cmdline.split():
+ # It can be that perf was attached to PID and was still running in
+ # system-wide mode.
+ # We filter out this case here since it's not per-process.
+ pid_match = pid_regex.match(cmdline)
+ if pid_match:
+ pids.append(pid_match.group('pid'))
+ # Stop the search and move to the next perf.data file.
+ break
+ else:
+ # cmdline wasn't found in the header. It's a fatal error.
+ raise PerfDataReadError(f'Perf command line is not found in {path_str}')
+ return pids
+
+ def VerifyPerfDataPID(self):
+ """Verify PIDs in per-process perf.data profiles.
+
+ Check that at list one top process is profiled if perf was running in
+ per-process mode.
+
+ Raises:
+ PidVerificationError if PID verification of per-process perf.data profiles
+ fail.
+ """
+ perf_data_pids = self.ReadPidFromPerfData()
+ if not perf_data_pids:
+ # In system-wide mode there are no PIDs.
+ self._logger.LogOutput('System-wide perf mode. Skip verification.')
+ return
+
+ # PIDs will be present only in per-process profiles.
+ # In this case we need to verify that profiles are collected on the
+ # hottest processes.
+ top_processes = [top_cmd['cmd'] for top_cmd in self.top_cmds]
+ # top_process structure: <cmd>-<pid>
+ top_pids = [top_process.split('-')[-1] for top_process in top_processes]
+ for top_pid in top_pids:
+ if top_pid in perf_data_pids:
+ self._logger.LogOutput('PID verification passed! '
+ f'Top process {top_pid} is profiled.')
+ return
+ raise PidVerificationError(
+ f'top processes {top_processes} are missing in perf.data traces with'
+ f' PID: {perf_data_pids}.')
+
def ProcessResults(self, use_cache=False):
# Note that this function doesn't know anything about whether there is a
# cache hit or miss. It should process results agnostic of the cache hit
@@ -869,6 +972,9 @@ class Result(object):
cpustats = self.ProcessCpustatsResults()
if self.top_log_file:
self.top_cmds = self.ProcessTopResults()
+ # Verify that PID in non system-wide perf.data and top_cmds are matching.
+ if self.perf_data_files and self.top_cmds:
+ self.VerifyPerfDataPID()
if self.wait_time_log_file:
with open(self.wait_time_log_file) as f:
wait_time = f.readline().strip()
@@ -949,8 +1055,8 @@ class Result(object):
def CreateTarball(self, results_dir, tarball):
if not results_dir.strip():
- raise ValueError(
- 'Refusing to `tar` an empty results_dir: %r' % results_dir)
+ raise ValueError('Refusing to `tar` an empty results_dir: %r' %
+ results_dir)
ret = self.ce.RunCommand('cd %s && '
'tar '
@@ -1000,8 +1106,8 @@ class Result(object):
if ret:
command = 'rm -rf {0}'.format(temp_dir)
self.ce.RunCommand(command)
- raise RuntimeError(
- 'Could not move dir %s to dir %s' % (temp_dir, cache_dir))
+ raise RuntimeError('Could not move dir %s to dir %s' %
+ (temp_dir, cache_dir))
@classmethod
def CreateFromRun(cls,
diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py
index 2c1996c4..91ceed22 100755
--- a/crosperf/results_cache_unittest.py
+++ b/crosperf/results_cache_unittest.py
@@ -21,6 +21,8 @@ import test_flag
from label import MockLabel
from results_cache import CacheConditions
+from results_cache import PerfDataReadError
+from results_cache import PidVerificationError
from results_cache import Result
from results_cache import ResultsCache
from results_cache import TelemetryResult
@@ -158,6 +160,34 @@ keyvals = {
'b_string_strstr___abcdefghijklmnopqrstuvwxyz__': '0.0134553343333'
}
+PERF_DATA_HEADER = """
+# ========
+# captured on : Thu Jan 01 00:00:00 1980
+# header version : 1
+# data offset : 536
+# data size : 737678672
+# feat offset : 737679208
+# hostname : localhost
+# os release : 5.4.61
+# perf version :
+# arch : aarch64
+# nrcpus online : 8
+# nrcpus avail : 8
+# total memory : 5911496 kB
+# cmdline : /usr/bin/perf record -e instructions -p {pid}
+# event : name = instructions, , id = ( 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193 ), type = 8, size = 112
+# event : name = dummy:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9
+# CPU_TOPOLOGY info available, use -I to display
+# pmu mappings: software = 1, uprobe = 6, cs_etm = 8, breakpoint = 5, tracepoint = 2, armv8_pmuv3 = 7
+# contains AUX area data (e.g. instruction trace)
+# time of first sample : 0.000000
+# time of last sample : 0.000000
+# sample duration : 0.000 ms
+# missing features: TRACING_DATA CPUDESC CPUID NUMA_TOPOLOGY BRANCH_STACK GROUP_DESC STAT CACHE MEM_TOPOLOGY CLOCKID DIR_FORMAT
+# ========
+#
+"""
+
TURBOSTAT_LOG_OUTPUT = \
"""CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ CoreTmp
- 329 12.13 2723 2393 10975 77
@@ -445,6 +475,7 @@ class ResultTest(unittest.TestCase):
self.callGetTurbostatFile = False
self.callGetCpustatsFile = False
self.callGetTopFile = False
+ self.callGetCpuinfoFile = False
self.callGetWaitTimeFile = False
self.args = None
self.callGatherPerfResults = False
@@ -469,6 +500,7 @@ class ResultTest(unittest.TestCase):
def setUp(self):
self.result = Result(self.mock_logger, self.mock_label, 'average',
self.mock_cmd_exec)
+ self.result.chromeos_root = '/tmp/chromeos'
@mock.patch.object(os.path, 'isdir')
@mock.patch.object(command_executer.CommandExecuter, 'RunCommand')
@@ -706,7 +738,8 @@ class ResultTest(unittest.TestCase):
self.assertEqual(mock_chrootruncmd.call_count, 1)
self.assertEqual(
mock_chrootruncmd.call_args_list[0][0],
- ('/tmp', ('./generate_test_report --no-color --csv %s') % TMP_DIR1))
+ (self.result.chromeos_root,
+ ('./generate_test_report --no-color --csv %s') % TMP_DIR1))
self.assertEqual(mock_getpath.call_count, 1)
self.assertEqual(mock_mkdtemp.call_count, 1)
self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']})
@@ -879,6 +912,15 @@ class ResultTest(unittest.TestCase):
self.assertEqual(found_no_logs, '')
@mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
+ def test_get_cpuinfo_file_finds_single_log(self, mock_runcmd):
+ """Expected behavior when a single cpuinfo file found."""
+ self.result.results_dir = '/tmp/test_results'
+ self.result.ce.RunCommandWOutput = mock_runcmd
+ mock_runcmd.return_value = (0, 'some/long/path/cpuinfo.log', '')
+ found_single_log = self.result.GetCpuinfoFile()
+ self.assertEqual(found_single_log, 'some/long/path/cpuinfo.log')
+
+ @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
def test_get_cpustats_file_finds_single_log(self, mock_runcmd):
"""Expected behavior when a single log file found."""
self.result.results_dir = '/tmp/test_results'
@@ -906,6 +948,101 @@ class ResultTest(unittest.TestCase):
found_no_logs = self.result.GetCpustatsFile()
self.assertEqual(found_no_logs, '')
+ def test_verify_perf_data_pid_ok(self):
+ """Verify perf PID which is present in TOP_DATA."""
+ self.result.top_cmds = TOP_DATA
+ # pid is present in TOP_DATA.
+ with mock.patch.object(
+ Result, 'ReadPidFromPerfData', return_value=['5713']):
+ self.result.VerifyPerfDataPID()
+
+ def test_verify_perf_data_pid_fail(self):
+ """Test perf PID missing in top raises the error."""
+ self.result.top_cmds = TOP_DATA
+ # pid is not in the list of top processes.
+ with mock.patch.object(
+ Result, 'ReadPidFromPerfData', return_value=['9999']):
+ with self.assertRaises(PidVerificationError):
+ self.result.VerifyPerfDataPID()
+
+ @mock.patch.object(command_executer.CommandExecuter,
+ 'ChrootRunCommandWOutput')
+ def test_read_pid_from_perf_data_ok(self, mock_runcmd):
+ """Test perf header parser, normal flow."""
+ self.result.ce.ChrootRunCommandWOutput = mock_runcmd
+ self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data']
+ exp_pid = '12345'
+ mock_runcmd.return_value = (0, PERF_DATA_HEADER.format(pid=exp_pid), '')
+ pids = self.result.ReadPidFromPerfData()
+ self.assertEqual(pids, [exp_pid])
+
+ @mock.patch.object(command_executer.CommandExecuter,
+ 'ChrootRunCommandWOutput')
+ def test_read_pid_from_perf_data_mult_profiles(self, mock_runcmd):
+ """Test multiple perf.data files with PID."""
+ self.result.ce.ChrootRunCommandWOutput = mock_runcmd
+ # self.result.chromeos_root = '/tmp/chromeos'
+ self.result.perf_data_files = [
+ '/tmp/chromeos/chroot/tmp/results/perf.data.0',
+ '/tmp/chromeos/chroot/tmp/results/perf.data.1',
+ ]
+ # There is '-p <pid>' in command line but it's still system-wide: '-a'.
+ cmd_line = '# cmdline : /usr/bin/perf record -e instructions -p {pid}'
+ exp_perf_pids = ['1111', '2222']
+ mock_runcmd.side_effect = [
+ (0, cmd_line.format(pid=exp_perf_pids[0]), ''),
+ (0, cmd_line.format(pid=exp_perf_pids[1]), ''),
+ ]
+ pids = self.result.ReadPidFromPerfData()
+ self.assertEqual(pids, exp_perf_pids)
+
+ @mock.patch.object(command_executer.CommandExecuter,
+ 'ChrootRunCommandWOutput')
+ def test_read_pid_from_perf_data_no_pid(self, mock_runcmd):
+ """Test perf.data without PID."""
+ self.result.ce.ChrootRunCommandWOutput = mock_runcmd
+ self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data']
+ cmd_line = '# cmdline : /usr/bin/perf record -e instructions'
+ mock_runcmd.return_value = (0, cmd_line, '')
+ pids = self.result.ReadPidFromPerfData()
+ # pids is empty.
+ self.assertEqual(pids, [])
+
+ @mock.patch.object(command_executer.CommandExecuter,
+ 'ChrootRunCommandWOutput')
+ def test_read_pid_from_perf_data_system_wide(self, mock_runcmd):
+ """Test reading from system-wide profile with PID."""
+ self.result.ce.ChrootRunCommandWOutput = mock_runcmd
+ self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data']
+ # There is '-p <pid>' in command line but it's still system-wide: '-a'.
+ cmd_line = '# cmdline : /usr/bin/perf record -e instructions -a -p 1234'
+ mock_runcmd.return_value = (0, cmd_line, '')
+ pids = self.result.ReadPidFromPerfData()
+ # pids should be empty since it's not a per-process profiling.
+ self.assertEqual(pids, [])
+
+ @mock.patch.object(command_executer.CommandExecuter,
+ 'ChrootRunCommandWOutput')
+ def test_read_pid_from_perf_data_read_fail(self, mock_runcmd):
+ """Failure to read perf.data raises the error."""
+ self.result.ce.ChrootRunCommandWOutput = mock_runcmd
+ self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data']
+ # Error status of the profile read.
+ mock_runcmd.return_value = (1, '', '')
+ with self.assertRaises(PerfDataReadError):
+ self.result.ReadPidFromPerfData()
+
+ @mock.patch.object(command_executer.CommandExecuter,
+ 'ChrootRunCommandWOutput')
+ def test_read_pid_from_perf_data_fail(self, mock_runcmd):
+ """Failure to find cmdline in perf.data header raises the error."""
+ self.result.ce.ChrootRunCommandWOutput = mock_runcmd
+ self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data']
+ # Empty output.
+ mock_runcmd.return_value = (0, '', '')
+ with self.assertRaises(PerfDataReadError):
+ self.result.ReadPidFromPerfData()
+
def test_process_turbostat_results_with_valid_data(self):
"""Normal case when log exists and contains valid data."""
self.result.turbostat_log_file = '/tmp/somelogfile.log'
@@ -1102,10 +1239,11 @@ class ResultTest(unittest.TestCase):
# Debug path not found
self.result.label.debug_path = ''
tmp = self.result.GeneratePerfReportFiles()
- self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file])
+ self.assertEqual(tmp, ['/tmp/chromeos/chroot%s' % fake_file])
self.assertEqual(mock_chrootruncmd.call_args_list[0][0],
- ('/tmp', ('/usr/sbin/perf report -n '
- '-i %s --stdio > %s') % (fake_file, fake_file)))
+ (self.result.chromeos_root,
+ ('/usr/sbin/perf report -n '
+ '-i %s --stdio > %s') % (fake_file, fake_file)))
@mock.patch.object(misc, 'GetInsideChrootPath')
@mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand')
@@ -1120,11 +1258,12 @@ class ResultTest(unittest.TestCase):
# Debug path found
self.result.label.debug_path = '/tmp/debug'
tmp = self.result.GeneratePerfReportFiles()
- self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file])
+ self.assertEqual(tmp, ['/tmp/chromeos/chroot%s' % fake_file])
self.assertEqual(mock_chrootruncmd.call_args_list[0][0],
- ('/tmp', ('/usr/sbin/perf report -n --symfs /tmp/debug '
- '--vmlinux /tmp/debug/boot/vmlinux '
- '-i %s --stdio > %s') % (fake_file, fake_file)))
+ (self.result.chromeos_root,
+ ('/usr/sbin/perf report -n --symfs /tmp/debug '
+ '--vmlinux /tmp/debug/boot/vmlinux '
+ '-i %s --stdio > %s') % (fake_file, fake_file)))
@mock.patch.object(misc, 'GetOutsideChrootPath')
def test_populate_from_run(self, mock_getpath):
@@ -1157,6 +1296,10 @@ class ResultTest(unittest.TestCase):
self.callGetTopFile = True
return []
+ def FakeGetCpuinfoFile():
+ self.callGetCpuinfoFile = True
+ return []
+
def FakeGetWaitTimeFile():
self.callGetWaitTimeFile = True
return []
@@ -1169,7 +1312,6 @@ class ResultTest(unittest.TestCase):
if mock_getpath:
pass
mock.get_path = '/tmp/chromeos/tmp/results_dir'
- self.result.chromeos_root = '/tmp/chromeos'
self.callGetResultsDir = False
self.callGetResultsFile = False
@@ -1178,6 +1320,7 @@ class ResultTest(unittest.TestCase):
self.callGetTurbostatFile = False
self.callGetCpustatsFile = False
self.callGetTopFile = False
+ self.callGetCpuinfoFile = False
self.callGetWaitTimeFile = False
self.callProcessResults = False
@@ -1188,6 +1331,7 @@ class ResultTest(unittest.TestCase):
self.result.GetTurbostatFile = FakeGetTurbostatFile
self.result.GetCpustatsFile = FakeGetCpustatsFile
self.result.GetTopFile = FakeGetTopFile
+ self.result.GetCpuinfoFile = FakeGetCpuinfoFile
self.result.GetWaitTimeFile = FakeGetWaitTimeFile
self.result.ProcessResults = FakeProcessResults
@@ -1200,6 +1344,7 @@ class ResultTest(unittest.TestCase):
self.assertTrue(self.callGetTurbostatFile)
self.assertTrue(self.callGetCpustatsFile)
self.assertTrue(self.callGetTopFile)
+ self.assertTrue(self.callGetCpuinfoFile)
self.assertTrue(self.callGetWaitTimeFile)
self.assertTrue(self.callProcessResults)
@@ -1436,8 +1581,7 @@ class ResultTest(unittest.TestCase):
u'crypto-md5__crypto-md5': [10.5, u'ms'],
u'string-tagcloud__string-tagcloud': [52.8, u'ms'],
u'access-nbody__access-nbody': [8.5, u'ms'],
- 'retval':
- 0,
+ 'retval': 0,
u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'],
u'math-cordic__math-cordic': [8.7, u'ms'],
u'access-binary-trees__access-binary-trees': [4.5, u'ms'],
@@ -1475,8 +1619,7 @@ class ResultTest(unittest.TestCase):
u'crypto-md5__crypto-md5': [10.5, u'ms'],
u'string-tagcloud__string-tagcloud': [52.8, u'ms'],
u'access-nbody__access-nbody': [8.5, u'ms'],
- 'retval':
- 0,
+ 'retval': 0,
u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'],
u'math-cordic__math-cordic': [8.7, u'ms'],
u'access-binary-trees__access-binary-trees': [4.5, u'ms'],
@@ -1705,8 +1848,9 @@ class TelemetryResultTest(unittest.TestCase):
'autotest_dir', 'debug_dir', '/tmp', 'lumpy',
'remote', 'image_args', 'cache_dir', 'average',
'gcc', False, None)
- self.mock_machine = machine_manager.MockCrosMachine(
- 'falco.cros', '/tmp/chromeos', 'average')
+ self.mock_machine = machine_manager.MockCrosMachine('falco.cros',
+ '/tmp/chromeos',
+ 'average')
def test_populate_from_run(self):
@@ -1786,10 +1930,12 @@ class ResultsCacheTest(unittest.TestCase):
def FakeGetMachines(label):
if label:
pass
- m1 = machine_manager.MockCrosMachine(
- 'lumpy1.cros', self.results_cache.chromeos_root, 'average')
- m2 = machine_manager.MockCrosMachine(
- 'lumpy2.cros', self.results_cache.chromeos_root, 'average')
+ m1 = machine_manager.MockCrosMachine('lumpy1.cros',
+ self.results_cache.chromeos_root,
+ 'average')
+ m2 = machine_manager.MockCrosMachine('lumpy2.cros',
+ self.results_cache.chromeos_root,
+ 'average')
return [m1, m2]
mock_checksum.return_value = 'FakeImageChecksumabc123'
@@ -1831,10 +1977,12 @@ class ResultsCacheTest(unittest.TestCase):
def FakeGetMachines(label):
if label:
pass
- m1 = machine_manager.MockCrosMachine(
- 'lumpy1.cros', self.results_cache.chromeos_root, 'average')
- m2 = machine_manager.MockCrosMachine(
- 'lumpy2.cros', self.results_cache.chromeos_root, 'average')
+ m1 = machine_manager.MockCrosMachine('lumpy1.cros',
+ self.results_cache.chromeos_root,
+ 'average')
+ m2 = machine_manager.MockCrosMachine('lumpy2.cros',
+ self.results_cache.chromeos_root,
+ 'average')
return [m1, m2]
mock_checksum.return_value = 'FakeImageChecksumabc123'
diff --git a/crosperf/results_report.py b/crosperf/results_report.py
index ff6c4f96..dc80b53b 100644
--- a/crosperf/results_report.py
+++ b/crosperf/results_report.py
@@ -418,8 +418,8 @@ class TextResultsReport(ResultsReport):
cpu_info = experiment.machine_manager.GetAllCPUInfo(experiment.labels)
sections.append(self._MakeSection('CPUInfo', cpu_info))
- totaltime = (
- time.time() - experiment.start_time) if experiment.start_time else 0
+ totaltime = (time.time() -
+ experiment.start_time) if experiment.start_time else 0
totaltime_str = 'Total experiment time:\n%d min' % (totaltime // 60)
cooldown_waittime_list = ['Cooldown wait time:']
# When running experiment on multiple DUTs cooldown wait time may vary
@@ -430,8 +430,9 @@ class TextResultsReport(ResultsReport):
cooldown_waittime_list.append('DUT %s: %d min' % (dut, waittime // 60))
cooldown_waittime_str = '\n'.join(cooldown_waittime_list)
sections.append(
- self._MakeSection('Duration', '\n\n'.join(
- [totaltime_str, cooldown_waittime_str])))
+ self._MakeSection('Duration',
+ '\n\n'.join([totaltime_str,
+ cooldown_waittime_str])))
return '\n'.join(sections)
@@ -505,7 +506,7 @@ class HTMLResultsReport(ResultsReport):
experiment_file = ''
if self.experiment is not None:
experiment_file = self.experiment.experiment_file
- # Use kwargs for sanity, and so that testing is a bit easier.
+ # Use kwargs for code readability, and so that testing is a bit easier.
return templates.GenerateHTMLPage(
perf_table=perf_table,
chart_js=chart_javascript,
diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py
index e03ea431..1e96ef97 100755
--- a/crosperf/results_report_unittest.py
+++ b/crosperf/results_report_unittest.py
@@ -57,7 +57,8 @@ class FreeFunctionsTest(unittest.TestCase):
ParseChromeosImage(os.path.dirname(buildbot_case)),
('', os.path.dirname(buildbot_img)))
- # Ensure we don't act completely insanely given a few mildly insane paths.
+ # Ensure we do something reasonable when giving paths that don't quite
+ # match the expected pattern.
fun_case = '/chromiumos_test_image.bin'
self.assertEqual(ParseChromeosImage(fun_case), ('', fun_case))
diff --git a/debug_info_test/whitelist.py b/debug_info_test/allowlist.py
index b53387a8..9cf42af0 100644
--- a/debug_info_test/whitelist.py
+++ b/debug_info_test/allowlist.py
@@ -3,12 +3,12 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""Whitelist functions."""
+"""Allowlist functions."""
from __future__ import print_function
-import os
import glob
+import os
import re
@@ -18,20 +18,20 @@ import re
#
# The performance bottleneck of this script is readelf. Unless this becomes
# slower than readelf, don't waste time here.
-def is_whitelisted(list_name, pattern):
- """Check whether the given pattern is specified in the whitelist.
+def is_allowlisted(list_name, pattern):
+ """Check whether the given pattern is specified in the allowlist.
Args:
- list_name: name of the whitelist.
+ list_name: name of the allowlist.
pattern: the target string.
Returns:
True if matched otherwise False.
"""
- return pattern and whitelists[list_name].match(pattern)
+ return pattern and allowlists[list_name].match(pattern)
-def prepare_whitelist(patterns):
+def prepare_allowlist(patterns):
"""Join and compile the re patterns.
Args:
@@ -43,26 +43,26 @@ def prepare_whitelist(patterns):
return re.compile('|'.join(patterns))
-def load_whitelists(dirname):
- """Load whitelists under dirname.
+def load_allowlists(dirname):
+ """Load allowlists under dirname.
- A whitelist ends with .whitelist.
+ An allowlist ends with .allowlist.
Args:
dirname: path to the dir.
Returns:
- A dictionary of 'filename' -> whitelist matcher.
+ A dictionary of 'filename' -> allowlist matcher.
"""
wlist = {}
- for fn in glob.glob(os.path.join(dirname, '*.whitelist')):
+ for fn in glob.glob(os.path.join(dirname, '*.allowlist')):
key = os.path.splitext(os.path.basename(fn))[0]
with open(fn, 'r', encoding='utf-8') as f:
patterns = f.read().splitlines()
patterns = [l for l in patterns if l != '']
patterns = [l for l in patterns if l[0] != '#']
- wlist[key] = prepare_whitelist(patterns)
+ wlist[key] = prepare_allowlist(patterns)
return wlist
-whitelists = load_whitelists(os.path.dirname(__file__))
+allowlists = load_allowlists(os.path.dirname(__file__))
diff --git a/debug_info_test/check_exist.py b/debug_info_test/check_exist.py
index dbb89127..f2cc7c6b 100644
--- a/debug_info_test/check_exist.py
+++ b/debug_info_test/check_exist.py
@@ -10,7 +10,7 @@ from __future__ import print_function
import os
import subprocess
-from whitelist import is_whitelisted
+from allowlist import is_allowlisted
def check_debug_info(dso_path, readelf_content):
@@ -24,8 +24,8 @@ def check_debug_info(dso_path, readelf_content):
True if debug info section exists, otherwise False.
"""
- # Return True if it is whitelisted
- if is_whitelisted('exist_debug_info', dso_path):
+ # Return True if it is allowlisted
+ if is_allowlisted('exist_debug_info', dso_path):
return True
for l in readelf_content:
@@ -46,8 +46,8 @@ def check_producer(dso_path, readelf_content):
Notice: If no compile unit in DSO, also return True.
"""
- # Return True if it is whitelisted
- if is_whitelisted('exist_producer', dso_path):
+ # Return True if it is allowlisted
+ if is_allowlisted('exist_producer', dso_path):
return True
# Indicate if there is a producer under each cu
diff --git a/debug_info_test/check_ngcc.py b/debug_info_test/check_ngcc.py
index 501bb988..c86c220a 100644
--- a/debug_info_test/check_ngcc.py
+++ b/debug_info_test/check_ngcc.py
@@ -7,7 +7,7 @@
from __future__ import print_function
-from whitelist import is_whitelisted
+from allowlist import is_allowlisted
def not_by_gcc(dso_path, producer, comp_path):
@@ -21,10 +21,10 @@ def not_by_gcc(dso_path, producer, comp_path):
Returns:
False if compiled by gcc otherwise True.
"""
- if is_whitelisted('ngcc_comp_path', comp_path):
+ if is_allowlisted('ngcc_comp_path', comp_path):
return True
- if is_whitelisted('ngcc_dso_path', dso_path):
+ if is_allowlisted('ngcc_dso_path', dso_path):
return True
return 'GNU C' not in producer
diff --git a/debug_info_test/exist_debug_info.whitelist b/debug_info_test/exist_debug_info.allowlist
index e0076fd6..e0076fd6 100644
--- a/debug_info_test/exist_debug_info.whitelist
+++ b/debug_info_test/exist_debug_info.allowlist
diff --git a/debug_info_test/exist_producer.whitelist b/debug_info_test/exist_producer.allowlist
index ee75de72..ee75de72 100644
--- a/debug_info_test/exist_producer.whitelist
+++ b/debug_info_test/exist_producer.allowlist
diff --git a/debug_info_test/ngcc_comp_path.whitelist b/debug_info_test/ngcc_comp_path.allowlist
index 45c5b4a2..45c5b4a2 100644
--- a/debug_info_test/ngcc_comp_path.whitelist
+++ b/debug_info_test/ngcc_comp_path.allowlist
diff --git a/debug_info_test/ngcc_dso_path.whitelist b/debug_info_test/ngcc_dso_path.allowlist
index 8d63a521..858465e0 100644
--- a/debug_info_test/ngcc_dso_path.whitelist
+++ b/debug_info_test/ngcc_dso_path.allowlist
@@ -1,5 +1,5 @@
# DSOs specified here are not CrOS packages compiled within CrOS SDK.
-# CrOS packages should be whitelisted in *_comp_path.whitelist
+# CrOS packages should be allowlisted in *_comp_path.allowlist
# modules we don't care:
.*/binutils/.*
.*/binutils-bin/.*
diff --git a/llvm_tools/README.md b/llvm_tools/README.md
index 1c1862d8..783ec22d 100644
--- a/llvm_tools/README.md
+++ b/llvm_tools/README.md
@@ -278,6 +278,7 @@ $ ./auto_llvm_bisection.py --start_rev 369410 --end_rev 369420 \
--last_tested /abs/path/to/last_tested_file.json \
--extra_change_lists 513590 1394249 \
--options latest-toolchain nochromesdk \
+ --chroot_path /path/to/chromeos/chroot \
--builder eve-release-tryjob
```
@@ -488,7 +489,7 @@ these synthesized numbers and git SHAs. Usage should be straightforward:
6f635f90929da9545dd696071a829a1a42f84b30
~> ./git_llvm_rev.py --llvm_dir llvm-project-copy/ --sha 6f635f90929da9545dd696071a829a1a42f84b30
r380000
-~> ./git_llvm_rev.py --llvm_dir llvm-project-copy/ --sha origin/master
+~> ./git_llvm_rev.py --llvm_dir llvm-project-copy/ --sha origin/some-branch
r387778
```
@@ -550,3 +551,44 @@ PYTHONPATH=../ ./nightly_revert_checker.py \
--llvm_dir llvm-project-copy \
--chromeos_dir ../../../../
```
+
+### `bisect_clang_crashes.py`
+
+This script downloads clang crash diagnoses from
+gs://chromeos-toolchain-artifacts/clang-crash-diagnoses and sends them to 4c for
+bisection.
+
+Usage example:
+
+```
+$ ./bisect_clang_crashes.py --4c 4c-cli --state_file ./output/state.json
+```
+
+The above command downloads the artifacts of clang crash diagnoses and send them
+to 4c server for bisection. The summary of submitted jobs will be saved in
+output/state.json under the current path. The output directory will be created
+automatically if it does not exist yet. To get more information of the submitted
+jobs, please refer to go/4c-cli.
+
+### `upload_lexan_crashes_to_forcey.py`
+
+This script downloads clang crash diagnoses from Lexan's bucket and sends them
+to 4c for bisection.
+
+Usage example:
+
+```
+$ ./upload_lexan_crashes_to_forcey.py --4c 4c-cli \
+ --state_file ./output/state.json
+```
+
+The above command downloads the artifacts of clang crash diagnoses and send them
+to 4c server for bisection. The summary of submitted jobs will be saved in
+output/state.json under the current path. The output directory will be created
+automatically if it does not exist yet. To get more information of the submitted
+jobs, please refer to go/4c-cli.
+
+Note that it's recommended to 'seed' the state file with a most recent upload
+date. This can be done by running this tool *once* with a `--last_date` flag.
+This flag has the script override whatever's in the state file (if anything) and
+start submitting all crashes uploaded starting at the given day.
diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py
new file mode 100755
index 00000000..e8ee2ab6
--- /dev/null
+++ b/llvm_tools/bisect_clang_crashes.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches and submits the artifacts from Chrome OS toolchain's crash bucket.
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import glob
+import json
+import logging
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+
+import chroot
+
+
+def get_artifacts(pattern):
+ results = subprocess.check_output(['gsutil.py', 'ls', pattern],
+ stderr=subprocess.STDOUT,
+ encoding='utf-8')
+ return sorted(l.strip() for l in results.splitlines())
+
+
+def get_crash_reproducers(working_dir):
+ results = []
+ for src in [
+ f for f in glob.glob('%s/*.c*' % working_dir)
+ if f.split('.')[-1] in ['c', 'cc', 'cpp']
+ ]:
+ script = '.'.join(src.split('.')[:-1]) + '.sh'
+ if not os.path.exists(script):
+ logging.warning('could not find the matching script of %s', src)
+ else:
+ results.append((src, script))
+ return results
+
+
+def submit_crash_to_forcey(forcey: str, temporary_directory: str,
+ buildbucket_id: str, url: str) -> None:
+ dest_dir = os.path.join(temporary_directory, buildbucket_id)
+ dest_file = os.path.join(dest_dir, os.path.basename(url))
+ logging.info('Downloading and submitting %r...', url)
+ subprocess.check_output(['gsutil.py', 'cp', url, dest_file],
+ stderr=subprocess.STDOUT)
+ subprocess.check_output(['tar', '-xJf', dest_file], cwd=dest_dir)
+ for src, script in get_crash_reproducers(dest_dir):
+ subprocess.check_output([
+ forcey, 'reduce', '-wait=false', '-note',
+ '%s:%s' % (url, src), '-sh_file', script, '-src_file', src
+ ])
+
+
+def main(argv):
+ chroot.VerifyOutsideChroot()
+ logging.basicConfig(
+ format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
+ level=logging.INFO,
+ )
+ cur_dir = os.path.dirname(os.path.abspath(__file__))
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ '--4c', dest='forcey', required=True, help='Path to a 4c client binary')
+ parser.add_argument(
+ '--state_file',
+ default=os.path.join(cur_dir, 'chromeos-state.json'),
+ help='The path to the state file.')
+ parser.add_argument(
+ '--nocleanup',
+ action='store_false',
+ dest='cleanup',
+ help='Keep temporary files created after the script finishes.')
+ opts = parser.parse_args(argv)
+
+ state_file = os.path.abspath(opts.state_file)
+ os.makedirs(os.path.dirname(state_file), exist_ok=True)
+ temporary_directory = '/tmp/bisect_clang_crashes'
+ os.makedirs(temporary_directory, exist_ok=True)
+ urls = get_artifacts('gs://chromeos-toolchain-artifacts/clang-crash-diagnoses'
+ '/**/*clang_crash_diagnoses.tar.xz')
+ logging.info('%d crash URLs found', len(urls))
+
+ visited = {}
+ if os.path.exists(state_file):
+ buildbucket_ids = {url.split('/')[-2] for url in urls}
+ with open(state_file, encoding='utf-8') as f:
+ data = json.load(f)
+ visited = {k: v for k, v in data.items() if k in buildbucket_ids}
+ logging.info('Successfully loaded %d previously-submitted crashes',
+ len(visited))
+
+ try:
+ for url in urls:
+ splits = url.split('/')
+ buildbucket_id = splits[-2]
+ # Skip the builds that has been processed
+ if buildbucket_id in visited:
+ continue
+ submit_crash_to_forcey(
+ forcey=opts.forcey,
+ temporary_directory=temporary_directory,
+ buildbucket_id=buildbucket_id,
+ url=url,
+ )
+ visited[buildbucket_id] = url
+
+ exception_in_flight = False
+ except:
+ exception_in_flight = True
+ raise
+ finally:
+ if exception_in_flight:
+ # This is best-effort. If the machine powers off or similar, we'll just
+ # resubmit the same crashes, which is suboptimal, but otherwise
+ # acceptable.
+ logging.error('Something went wrong; attempting to save our work...')
+ else:
+ logging.info('Persisting state...')
+
+ tmp_state_file = state_file + '.tmp'
+ with open(tmp_state_file, 'w', encoding='utf-8') as f:
+ json.dump(visited, f, indent=2)
+ os.rename(tmp_state_file, state_file)
+
+ logging.info('State successfully persisted')
+
+ if opts.cleanup:
+ shutil.rmtree(temporary_directory)
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/llvm_tools/bisect_clang_crashes_unittest.py b/llvm_tools/bisect_clang_crashes_unittest.py
new file mode 100755
index 00000000..c9143450
--- /dev/null
+++ b/llvm_tools/bisect_clang_crashes_unittest.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for bisect_clang_crashes."""
+
+# pylint: disable=cros-logging-import
+import glob
+import logging
+import os.path
+import subprocess
+import unittest
+import unittest.mock as mock
+
+import bisect_clang_crashes
+
+
+class Test(unittest.TestCase):
+ """Tests for bisect_clang_crashes."""
+
+ class _SilencingFilter(object):
+ """Silences all log messages.
+
+ Also collects info about log messages that would've been emitted.
+ """
+
+ def __init__(self):
+ self.messages = []
+
+ def filter(self, record):
+ self.messages.append(record.getMessage())
+ return 0
+
+ @mock.patch.object(subprocess, 'check_output')
+ def test_get_artifacts(self, mock_gsutil_ls):
+ pattern = 'gs://chromeos-toolchain-artifacts/clang-crash-diagnoses/' \
+ '**/*clang_crash_diagnoses.tar.xz'
+ mock_gsutil_ls.return_value = 'artifact1\nartifact2\nartifact3'
+ results = bisect_clang_crashes.get_artifacts(pattern)
+ self.assertEqual(results, ['artifact1', 'artifact2', 'artifact3'])
+ mock_gsutil_ls.assert_called_once_with(['gsutil.py', 'ls', pattern],
+ stderr=subprocess.STDOUT,
+ encoding='utf-8')
+
+ @mock.patch.object(os.path, 'exists')
+ @mock.patch.object(glob, 'glob')
+ def test_get_crash_reproducers_succeed(self, mock_file_search,
+ mock_file_check):
+ working_dir = 'SomeDirectory'
+ mock_file_search.return_value = ['a.c', 'b.cpp', 'c.cc']
+ mock_file_check.side_effect = [True, True, True]
+ results = bisect_clang_crashes.get_crash_reproducers(working_dir)
+ mock_file_search.assert_called_once_with('%s/*.c*' % working_dir)
+ self.assertEqual(mock_file_check.call_count, 3)
+ self.assertEqual(mock_file_check.call_args_list[0], mock.call('a.sh'))
+ self.assertEqual(mock_file_check.call_args_list[1], mock.call('b.sh'))
+ self.assertEqual(mock_file_check.call_args_list[2], mock.call('c.sh'))
+ self.assertEqual(results, [('a.c', 'a.sh'), ('b.cpp', 'b.sh'),
+ ('c.cc', 'c.sh')])
+
+ @mock.patch.object(os.path, 'exists')
+ @mock.patch.object(glob, 'glob')
+ def test_get_crash_reproducers_no_matching_script(self, mock_file_search,
+ mock_file_check):
+
+ def silence_logging():
+ root = logging.getLogger()
+ filt = self._SilencingFilter()
+ root.addFilter(filt)
+ self.addCleanup(root.removeFilter, filt)
+ return filt
+
+ log_filter = silence_logging()
+ working_dir = 'SomeDirectory'
+ mock_file_search.return_value = ['a.c', 'b.cpp', 'c.cc']
+ mock_file_check.side_effect = [True, False, True]
+ results = bisect_clang_crashes.get_crash_reproducers(working_dir)
+ mock_file_search.assert_called_once_with('%s/*.c*' % working_dir)
+ self.assertEqual(mock_file_check.call_count, 3)
+ self.assertEqual(mock_file_check.call_args_list[0], mock.call('a.sh'))
+ self.assertEqual(mock_file_check.call_args_list[1], mock.call('b.sh'))
+ self.assertEqual(mock_file_check.call_args_list[2], mock.call('c.sh'))
+ self.assertEqual(results, [('a.c', 'a.sh'), ('c.cc', 'c.sh')])
+ self.assertTrue(
+ any('could not find the matching script of b.cpp' in x
+ for x in log_filter.messages), log_filter.messages)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/llvm_tools/cherrypick_cl.py b/llvm_tools/cherrypick_cl.py
index 78330d19..9e306725 100755
--- a/llvm_tools/cherrypick_cl.py
+++ b/llvm_tools/cherrypick_cl.py
@@ -170,32 +170,34 @@ def main():
help='Automatically create a CL if specified')
args = parser.parse_args()
- llvm_config = git_llvm_rev.LLVMConfig(
- remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools())
-
llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths(
args.chroot_path,
chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm']))[0]
+ llvm_symlink_dir = os.path.dirname(llvm_symlink)
+
+ git_status = subprocess.check_output(['git', 'status', '-s'],
+ cwd=llvm_symlink_dir,
+ encoding='utf-8')
+ if git_status:
+ raise ValueError('Uncommited changes detected in %s' %
+ os.path.dirname(os.path.dirname(llvm_symlink_dir)))
+
start_sha = args.start_sha
if start_sha == 'llvm':
- start_sha = parse_ebuild_for_assignment(
- os.path.dirname(llvm_symlink), 'LLVM_HASH')
+ start_sha = parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_HASH')
elif start_sha == 'llvm-next':
- start_sha = parse_ebuild_for_assignment(
- os.path.dirname(llvm_symlink), 'LLVM_NEXT_HASH')
+ start_sha = parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_NEXT_HASH')
logging.info('Base llvm hash == %s', start_sha)
+ llvm_config = git_llvm_rev.LLVMConfig(
+ remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools())
+
start_sha = resolve_llvm_ref(llvm_config.dir, start_sha)
start_rev = git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha)
if args.create_cl:
branch = 'cherry-pick'
- symlink = os.path.dirname(
- chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm'])[0])
- symlink = chroot.ConvertChrootPathsToAbsolutePaths(args.chroot_path,
- [symlink])[0]
- symlink_dir = os.path.dirname(symlink)
- git.CreateBranch(symlink_dir, branch)
+ git.CreateBranch(llvm_symlink_dir, branch)
symlinks_to_uprev = []
commit_messages = [
'llvm: cherry-pick CLs from upstream\n',
@@ -221,9 +223,9 @@ def main():
relative_patches_dir = 'cherry' if package == 'llvm' else ''
patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir)
logging.info('Cherrypicking %s (%s) into %s', rev, sha, package)
+
add_cherrypick(patches_json_path, patches_dir, relative_patches_dir,
start_rev, llvm_config.dir, rev, sha, package)
-
if args.create_cl:
symlinks_to_uprev.extend(symlinks)
commit_messages.extend([
@@ -240,8 +242,8 @@ def main():
for symlink in symlinks_to_uprev:
update_chromeos_llvm_hash.UprevEbuildSymlink(symlink)
subprocess.check_output(['git', 'add', '--all'], cwd=symlink_dir)
- git.UploadChanges(symlink_dir, branch, commit_messages)
- git.DeleteBranch(symlink_dir, branch)
+ git.UploadChanges(llvm_symlink_dir, branch, commit_messages)
+ git.DeleteBranch(llvm_symlink_dir, branch)
if __name__ == '__main__':
diff --git a/llvm_tools/fetch_cros_sdk_rolls.py b/llvm_tools/fetch_cros_sdk_rolls.py
new file mode 100755
index 00000000..42af678a
--- /dev/null
+++ b/llvm_tools/fetch_cros_sdk_rolls.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Gets info about completed chromiumos-sdk runs.
+
+Moreover, this script exists to get versions of published sdk tarballs in
+gs://chromiumos-sdk/. The hope is that it'll help answer the question "when did
+the toolchain ebuild ${x} go live?"
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Dict, List
+from pathlib import Path
+
+
+def fetch_all_sdk_manifest_paths() -> List[str]:
+ """Fetches all paths of SDK manifests; newer = later in the return value."""
+ results = subprocess.run(
+ ['gsutil', 'ls', 'gs://chromiumos-sdk/cros-sdk-20??.*.Manifest'],
+ check=True,
+ stdout=subprocess.PIPE,
+ encoding='utf-8',
+ ).stdout
+ # These are named so that sorted order == newest last.
+ return sorted(x.strip() for x in results.splitlines())
+
+
+def fetch_manifests_into(into_dir: Path, manifests: List[str]):
+ # Wrap this in a `try` block because gsutil likes to print to stdout *and*
+ # stderr even on success, so we silence them & only print on failure.
+ try:
+ subprocess.run(
+ [
+ 'gsutil',
+ '-m',
+ 'cp',
+ '-I',
+ str(into_dir),
+ ],
+ check=True,
+ input='\n'.join(manifests),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding='utf-8',
+ )
+ except subprocess.CalledProcessError as e:
+ logging.exception('gsutil failed; output:\n%s', e.stdout)
+
+
+def load_manifest_versions(manifest: Path) -> Dict[str, str]:
+ with manifest.open(encoding='utf-8') as f:
+ raw_versions = json.load(f)
+
+ # We get a dict of list of lists of versions and some other metadata, e.g.
+ # {"foo/bar": [["1.2.3", {}]]}
+ # Trim out the metadata.
+ return {k: v[0][0] for k, v in raw_versions['packages'].items()}
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument(
+ '-d', '--debug', action='store_true', help='Emit debugging output')
+ parser.add_argument(
+ '-n',
+ '--number',
+ type=int,
+ default=20,
+ help='Number of recent manifests to fetch info about. 0 means unlimited.')
+ args = parser.parse_args()
+
+ is_debug = args.debug
+ logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO)
+
+ logging.debug('Fetching SDK manifests')
+ manifest_paths = fetch_all_sdk_manifest_paths()
+ logging.debug('%d SDK manifests fetched', len(manifest_paths))
+
+ number = args.number
+ if number:
+ manifest_paths = manifest_paths[-number:]
+
+ tempdir = Path(tempfile.mkdtemp(prefix='cros-sdk-rolls'))
+ try:
+ logging.debug('Working in tempdir %r', tempdir)
+ fetch_manifests_into(tempdir, manifest_paths)
+
+ for path in manifest_paths:
+ basename = os.path.basename(path)
+ versions = load_manifest_versions(tempdir.joinpath(basename))
+ print(f'{basename}: {versions["sys-devel/llvm"]}')
+ finally:
+ if is_debug:
+ logging.debug('Keeping around tempdir %r to aid debugging', tempdir)
+ else:
+ shutil.rmtree(tempdir)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py
index a5b5429e..4c479962 100755
--- a/llvm_tools/get_llvm_hash.py
+++ b/llvm_tools/get_llvm_hash.py
@@ -17,7 +17,8 @@ import tempfile
from contextlib import contextmanager
import git_llvm_rev
-from subprocess_helpers import CheckCommand, check_output
+from subprocess_helpers import CheckCommand
+from subprocess_helpers import check_output
_LLVM_GIT_URL = ('https://chromium.googlesource.com/external/github.com/llvm'
'/llvm-project')
@@ -39,7 +40,7 @@ def GetVersionFrom(src_dir, git_hash):
version = git_llvm_rev.translate_sha_to_rev(
git_llvm_rev.LLVMConfig(remote='origin', dir=src_dir), git_hash)
# Note: branches aren't supported
- assert version.branch == 'master', version.branch
+ assert version.branch == git_llvm_rev.MAIN_BRANCH, version.branch
return version.number
@@ -59,7 +60,7 @@ def GetGitHashFrom(src_dir, version):
return git_llvm_rev.translate_rev_to_sha(
git_llvm_rev.LLVMConfig(remote='origin', dir=src_dir),
- git_llvm_rev.Rev(branch='master', number=version))
+ git_llvm_rev.Rev(branch=git_llvm_rev.MAIN_BRANCH, number=version))
@contextmanager
@@ -85,13 +86,10 @@ def CreateTempLLVMRepo(temp_dir):
"""
abs_path_to_llvm_project_dir = GetAndUpdateLLVMProjectInLLVMTools()
-
- add_worktree_cmd = [
+ CheckCommand([
'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'add', '--detach',
- temp_dir, 'master'
- ]
-
- CheckCommand(add_worktree_cmd)
+ temp_dir, git_llvm_rev.MAIN_BRANCH
+ ])
try:
yield temp_dir
@@ -117,7 +115,7 @@ def GetAndUpdateLLVMProjectInLLVMTools():
Raises:
ValueError: LLVM repo (in 'llvm-project-copy' dir.) has changes or failed to
- checkout to master or failed to fetch from chromium mirror of LLVM.
+ checkout to main or failed to fetch from chromium mirror of LLVM.
"""
abs_path_to_llvm_tools_dir = os.path.dirname(os.path.abspath(__file__))
@@ -127,7 +125,8 @@ def GetAndUpdateLLVMProjectInLLVMTools():
if not os.path.isdir(abs_path_to_llvm_project_dir):
print(
- 'Checking out LLVM from scratch. This could take a while...',
+ 'Checking out LLVM from scratch. This could take a while...\n'
+ '(This should only need to be done once, though.)',
file=sys.stderr)
os.mkdir(abs_path_to_llvm_project_dir)
@@ -143,15 +142,11 @@ def GetAndUpdateLLVMProjectInLLVMTools():
raise ValueError('LLVM repo in %s has changes, please remove.' %
abs_path_to_llvm_project_dir)
- checkout_to_master_cmd = [
- 'git', '-C', abs_path_to_llvm_project_dir, 'checkout', 'master'
- ]
-
- CheckCommand(checkout_to_master_cmd)
-
- update_master_cmd = ['git', '-C', abs_path_to_llvm_project_dir, 'pull']
-
- CheckCommand(update_master_cmd)
+ CheckCommand([
+ 'git', '-C', abs_path_to_llvm_project_dir, 'checkout',
+ git_llvm_rev.MAIN_BRANCH
+ ])
+ CheckCommand(['git', '-C', abs_path_to_llvm_project_dir, 'pull'])
return abs_path_to_llvm_project_dir
@@ -298,14 +293,9 @@ class LLVMHash(object):
def GetTopOfTrunkGitHash(self):
"""Gets the latest git hash from top of trunk of LLVM."""
- path_to_master_branch = 'refs/heads/master'
-
- llvm_tot_git_hash_cmd = [
- 'git', 'ls-remote', _LLVM_GIT_URL, path_to_master_branch
- ]
-
- llvm_tot_git_hash = check_output(llvm_tot_git_hash_cmd)
-
+ path_to_main_branch = 'refs/heads/master'
+ llvm_tot_git_hash = check_output(
+ ['git', 'ls-remote', _LLVM_GIT_URL, path_to_main_branch])
return llvm_tot_git_hash.rstrip().split()[0]
diff --git a/llvm_tools/get_llvm_hash_unittest.py b/llvm_tools/get_llvm_hash_unittest.py
index c828f433..2e56aed5 100755
--- a/llvm_tools/get_llvm_hash_unittest.py
+++ b/llvm_tools/get_llvm_hash_unittest.py
@@ -8,12 +8,11 @@
from __future__ import print_function
-import get_llvm_hash
import subprocess
import unittest
import unittest.mock as mock
-import test_helpers
+import get_llvm_hash
from get_llvm_hash import LLVMHash
# We grab protected stuff from get_llvm_hash. That's OK.
@@ -87,7 +86,7 @@ class TestGetLLVMHash(unittest.TestCase):
@mock.patch.object(subprocess, 'check_output')
def testSuccessfullyGetGitHashFromToTOfLLVM(self, mock_check_output):
- mock_check_output.return_value = 'a123testhash1 path/to/master\n'
+ mock_check_output.return_value = 'a123testhash1 path/to/main\n'
self.assertEqual(LLVMHash().GetTopOfTrunkGitHash(), 'a123testhash1')
mock_check_output.assert_called_once()
diff --git a/llvm_tools/git_llvm_rev.py b/llvm_tools/git_llvm_rev.py
index 8eefcdce..07209f1e 100755
--- a/llvm_tools/git_llvm_rev.py
+++ b/llvm_tools/git_llvm_rev.py
@@ -18,6 +18,8 @@ import subprocess
import sys
import typing as t
+MAIN_BRANCH = 'master'
+
# Note that after base_llvm_sha, we reach The Wild West(TM) of commits.
# So reasonable input that could break us includes:
#
@@ -52,9 +54,9 @@ class Rev(t.NamedTuple('Rev', (('branch', str), ('number', int)))):
# pairs.
#
# We support r${commits_since_base_commit} as shorthand for
- # (master, r${commits_since_base_commit}).
+ # (main, r${commits_since_base_commit}).
if rev.startswith('r'):
- branch_name = 'master'
+ branch_name = MAIN_BRANCH
rev_string = rev[1:]
else:
match = re.match(r'\((.+), r(\d+)\)', rev)
@@ -67,7 +69,7 @@ class Rev(t.NamedTuple('Rev', (('branch', str), ('number', int)))):
def __str__(self) -> str:
branch_name, number = self
- if branch_name == 'master':
+ if branch_name == MAIN_BRANCH:
return 'r%d' % number
return '(%s, r%d)' % (branch_name, number)
@@ -141,7 +143,7 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev:
cwd=llvm_config.dir,
)
count = int(result.strip())
- return Rev(branch='master', number=count + base_llvm_revision)
+ return Rev(branch=MAIN_BRANCH, number=count + base_llvm_revision)
# Otherwise, either:
# - |merge_base| is |sha| (we have a guaranteed llvm-svn number on |sha|)
@@ -150,7 +152,7 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev:
merge_base_number = translate_prebase_sha_to_rev_number(
llvm_config, merge_base)
if merge_base == sha:
- return Rev(branch='master', number=merge_base_number)
+ return Rev(branch=MAIN_BRANCH, number=merge_base_number)
distance_from_base = check_output(
[
@@ -270,7 +272,7 @@ def translate_rev_to_sha(llvm_config: LLVMConfig, rev: Rev) -> str:
"""
branch, number = rev
- if branch == 'master':
+ if branch == MAIN_BRANCH:
if number < base_llvm_revision:
return translate_prebase_rev_to_sha(llvm_config, rev)
base_sha = base_llvm_sha
diff --git a/llvm_tools/git_llvm_rev_test.py b/llvm_tools/git_llvm_rev_test.py
index 1e38f589..74280c5d 100755
--- a/llvm_tools/git_llvm_rev_test.py
+++ b/llvm_tools/git_llvm_rev_test.py
@@ -6,12 +6,11 @@
"""Tests for git_llvm_rev."""
-from __future__ import print_function
-
import unittest
import git_llvm_rev
import llvm_project
+from git_llvm_rev import MAIN_BRANCH
def get_llvm_config() -> git_llvm_rev.LLVMConfig:
@@ -32,17 +31,17 @@ class Test(unittest.TestCase):
def test_sha_to_rev_on_base_sha_works(self) -> None:
sha = self.rev_to_sha_with_round_trip(
git_llvm_rev.Rev(
- branch='master', number=git_llvm_rev.base_llvm_revision))
+ branch=MAIN_BRANCH, number=git_llvm_rev.base_llvm_revision))
self.assertEqual(sha, git_llvm_rev.base_llvm_sha)
def test_sha_to_rev_prior_to_base_rev_works(self) -> None:
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=375000))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=375000))
self.assertEqual(sha, '2f6da767f13b8fd81f840c211d405fea32ac9db7')
def test_sha_to_rev_after_base_rev_works(self) -> None:
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=375506))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=375506))
self.assertEqual(sha, '3bf7fddeb05655d9baed4cc69e13535c677ed1dd')
def test_llvm_svn_parsing_runs_ignore_reverts(self) -> None:
@@ -51,18 +50,19 @@ class Test(unittest.TestCase):
# Commit which performed the revert
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=374895))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=374895))
self.assertEqual(sha, '1731fc88d1fa1fa55edd056db73a339b415dd5d6')
# Commit that was reverted
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=374841))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=374841))
self.assertEqual(sha, '2a1386c81de504b5bda44fbecf3f7b4cdfd748fc')
def test_imaginary_revs_raise(self) -> None:
with self.assertRaises(ValueError) as r:
git_llvm_rev.translate_rev_to_sha(
- get_llvm_config(), git_llvm_rev.Rev(branch='master', number=9999999))
+ get_llvm_config(),
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=9999999))
self.assertIn('Try updating your tree?', str(r.exception))
@@ -71,15 +71,15 @@ class Test(unittest.TestCase):
# properties about it.
merge_sha_rev_number = 4496 + git_llvm_rev.base_llvm_revision
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=merge_sha_rev_number))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=merge_sha_rev_number))
self.assertEqual(sha, '0f0d0ed1c78f1a80139a1f2133fad5284691a121')
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=merge_sha_rev_number - 1))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=merge_sha_rev_number - 1))
self.assertEqual(sha, '6f635f90929da9545dd696071a829a1a42f84b30')
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=merge_sha_rev_number + 1))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=merge_sha_rev_number + 1))
self.assertEqual(sha, '199700a5cfeedf227619f966aa3125cef18bc958')
# NOTE: The below tests have _zz_ in their name as an optimization. Iterating
@@ -101,11 +101,11 @@ class Test(unittest.TestCase):
backing_sha = 'c89a3d78f43d81b9cff7b9248772ddf14d21b749'
sha = self.rev_to_sha_with_round_trip(
- git_llvm_rev.Rev(branch='master', number=rev_number))
+ git_llvm_rev.Rev(branch=MAIN_BRANCH, number=rev_number))
self.assertEqual(sha, backing_sha)
- # Note that this won't round-trip: since this commit is on the master
- # branch, we'll pick master for this. That's fine
+ # Note that this won't round-trip: since this commit is on the main
+ # branch, we'll pick main for this. That's fine.
sha = git_llvm_rev.translate_rev_to_sha(
get_llvm_config(),
git_llvm_rev.Rev(branch='release/9.x', number=rev_number))
@@ -113,7 +113,7 @@ class Test(unittest.TestCase):
def test_zz_branch_revs_work_after_merge_points(self) -> None:
# Picking the commit on the 9.x branch after the merge-base for that +
- # master. Note that this is where llvm-svn numbers should diverge from
+ # main. Note that this is where llvm-svn numbers should diverge from
# ours, and are therefore untrustworthy. The commit for this *does* have a
# different `llvm-svn:` string than we should have.
sha = self.rev_to_sha_with_round_trip(
diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py
index 2772ca48..c8d694cd 100755
--- a/llvm_tools/llvm_bisection.py
+++ b/llvm_tools/llvm_bisection.py
@@ -13,10 +13,12 @@ import enum
import errno
import json
import os
+import subprocess
import sys
import chroot
import get_llvm_hash
+import git_llvm_rev
import modify_a_tryjob
import update_tryjob_status
@@ -28,11 +30,6 @@ class BisectionExitStatus(enum.Enum):
BISECTION_COMPLETE = 126
-def is_file_and_json(json_file):
- """Validates that the file exists and is a JSON file."""
- return os.path.isfile(json_file) and json_file.endswith('.json')
-
-
def GetCommandLineArgs():
"""Parses the command line for the command line arguments."""
@@ -121,30 +118,27 @@ def GetCommandLineArgs():
help='display contents of a command to the terminal '
'(default: %(default)s)')
+ # Add argument for whether to display command contents to `stdout`.
+ parser.add_argument(
+ '--nocleanup',
+ action='store_false',
+ dest='cleanup',
+ help='Abandon CLs created for bisectoin')
+
args_output = parser.parse_args()
assert args_output.start_rev < args_output.end_rev, (
- 'Start revision %d is >= end revision %d' % (args_output.start_rev,
- args_output.end_rev))
+ 'Start revision %d is >= end revision %d' %
+ (args_output.start_rev, args_output.end_rev))
if args_output.last_tested and not args_output.last_tested.endswith('.json'):
- raise ValueError(
- 'Filed provided %s does not end in ".json"' % args_output.last_tested)
+ raise ValueError('Filed provided %s does not end in ".json"' %
+ args_output.last_tested)
return args_output
-def _ValidateStartAndEndAgainstJSONStartAndEnd(start, end, json_start,
- json_end):
- """Valides that the command line arguments are the same as the JSON."""
-
- if start != json_start or end != json_end:
- raise ValueError('The start %d or the end %d version provided is '
- 'different than "start" %d or "end" %d in the .JSON '
- 'file' % (start, end, json_start, json_end))
-
-
-def GetStartAndEndRevision(start, end, tryjobs):
+def GetRemainingRange(start, end, tryjobs):
"""Gets the start and end intervals in 'json_file'.
Args:
@@ -230,145 +224,56 @@ def GetStartAndEndRevision(start, end, tryjobs):
return good_rev, bad_rev, pending_revisions, skip_revisions
-def GetRevisionsBetweenBisection(start, end, parallel, src_path,
- pending_revisions, skip_revisions):
- """Gets the revisions between 'start' and 'end'.
-
- Sometimes, the LLVM source tree's revisions do not increment by 1 (there is
- a jump), so need to construct a list of all revisions that are NOT missing
- between 'start' and 'end'. Then, the step amount (i.e. length of the list
- divided by ('parallel' + 1)) will be used for indexing into the list.
-
- Args:
- start: The start revision.
- end: The end revision.
- parallel: The number of tryjobs to create between 'start' and 'end'.
- src_path: The absolute path to the LLVM source tree to use.
- pending_revisions: A set containing 'pending' revisions that are between
- 'start' and 'end'.
- skip_revisions: A set containing revisions between 'start' and 'end' that
- are to be skipped.
-
- Returns:
- A list of revisions between 'start' and 'end'.
- """
-
- valid_revisions = []
-
- # Start at ('start' + 1) because 'start' is the good revision.
- #
- # FIXME: Searching for each revision from ('start' + 1) up to 'end' in the
- # LLVM source tree is a quadratic algorithm. It's a good idea to optimize
- # this.
- for cur_revision in range(start + 1, end):
- try:
- if cur_revision not in pending_revisions and \
- cur_revision not in skip_revisions:
- # Verify that the current revision exists by finding its corresponding
- # git hash in the LLVM source tree.
- get_llvm_hash.GetGitHashFrom(src_path, cur_revision)
- valid_revisions.append(cur_revision)
- except ValueError:
- # Could not find the git hash for the current revision.
- continue
-
- # ('parallel' + 1) so that the last revision in the list is not close to
- # 'end' (have a bit more coverage).
- index_step = len(valid_revisions) // (parallel + 1)
-
- if not index_step:
- index_step = 1
-
- result = [valid_revisions[index] \
- for index in range(0, len(valid_revisions), index_step)]
-
- return result
-
-
-def GetRevisionsListAndHashList(start, end, parallel, src_path,
- pending_revisions, skip_revisions):
+def GetCommitsBetween(start, end, parallel, src_path, pending_revisions,
+ skip_revisions):
"""Determines the revisions between start and end."""
- new_llvm = get_llvm_hash.LLVMHash()
+ with get_llvm_hash.LLVMHash().CreateTempDirectory() as temp_dir:
+ # We have guaranteed contiguous revision numbers after this,
+ # and that guarnatee simplifies things considerably, so we don't
+ # support anything before it.
+ assert start >= git_llvm_rev.base_llvm_revision, f'{start} was too long ago'
- with new_llvm.CreateTempDirectory() as temp_dir:
with get_llvm_hash.CreateTempLLVMRepo(temp_dir) as new_repo:
if not src_path:
src_path = new_repo
-
- # Get a list of revisions between start and end.
- revisions = GetRevisionsBetweenBisection(
- start, end, parallel, src_path, pending_revisions, skip_revisions)
-
+ index_step = (end - (start + 1)) // (parallel + 1)
+ if not index_step:
+ index_step = 1
+ revisions = [
+ rev for rev in range(start + 1, end, index_step)
+ if rev not in pending_revisions and rev not in skip_revisions
+ ]
git_hashes = [
get_llvm_hash.GetGitHashFrom(src_path, rev) for rev in revisions
]
+ return revisions, git_hashes
- return revisions, git_hashes
-
-
-def DieWithNoRevisionsError(start, end, skip_revisions, pending_revisions):
- """Raises a ValueError exception with useful information."""
-
- no_revisions_message = ('No revisions between start %d and end '
- '%d to create tryjobs' % (start, end))
- if pending_revisions:
- no_revisions_message += '\nThe following tryjobs are pending:\n' \
- + '\n'.join(str(rev) for rev in pending_revisions)
-
- if skip_revisions:
- no_revisions_message += '\nThe following tryjobs were skipped:\n' \
- + '\n'.join(str(rev) for rev in skip_revisions)
-
- raise ValueError(no_revisions_message)
-
-
-def CheckForExistingTryjobsInRevisionsToLaunch(revisions, jobs):
- """Checks if a revision in 'revisions' exists in 'jobs' list."""
-
- for rev in revisions:
- if update_tryjob_status.FindTryjobIndex(rev, jobs) is not None:
- raise ValueError('Revision %d exists already in "jobs"' % rev)
-
-
-def UpdateBisection(revisions, git_hashes, bisect_contents, last_tested,
- update_packages, chroot_path, patch_metadata_file,
- extra_change_lists, options, builder, verbose):
+def Bisect(revisions, git_hashes, bisect_state, last_tested, update_packages,
+ chroot_path, patch_metadata_file, extra_change_lists, options,
+ builder, verbose):
"""Adds tryjobs and updates the status file with the new tryjobs."""
try:
for svn_revision, git_hash in zip(revisions, git_hashes):
- tryjob_dict = modify_a_tryjob.AddTryjob(
- update_packages, git_hash, svn_revision, chroot_path,
- patch_metadata_file, extra_change_lists, options, builder, verbose,
- svn_revision)
+ tryjob_dict = modify_a_tryjob.AddTryjob(update_packages, git_hash,
+ svn_revision, chroot_path,
+ patch_metadata_file,
+ extra_change_lists, options,
+ builder, verbose, svn_revision)
- bisect_contents['jobs'].append(tryjob_dict)
+ bisect_state['jobs'].append(tryjob_dict)
finally:
# Do not want to lose progress if there is an exception.
if last_tested:
new_file = '%s.new' % last_tested
with open(new_file, 'w') as json_file:
- json.dump(bisect_contents, json_file, indent=4, separators=(',', ': '))
+ json.dump(bisect_state, json_file, indent=4, separators=(',', ': '))
os.rename(new_file, last_tested)
-def _NoteCompletedBisection(last_tested, src_path, end):
- """Prints that bisection is complete."""
-
- print('Finished bisecting for %s' % last_tested)
-
- if src_path:
- bad_llvm_hash = get_llvm_hash.GetGitHashFrom(src_path, end)
- else:
- bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end)
-
- print(
- 'The bad revision is %d and its commit hash is %s' % (end, bad_llvm_hash))
-
-
def LoadStatusFile(last_tested, start, end):
"""Loads the status file for bisection."""
@@ -383,37 +288,36 @@ def LoadStatusFile(last_tested, start, end):
def main(args_output):
- """Bisects LLVM based off of a .JSON file.
+ """Bisects LLVM commits.
Raises:
AssertionError: The script was run inside the chroot.
"""
chroot.VerifyOutsideChroot()
-
update_packages = [
'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx',
'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind'
]
-
patch_metadata_file = 'PATCHES.json'
-
start = args_output.start_rev
end = args_output.end_rev
- bisect_contents = LoadStatusFile(args_output.last_tested, start, end)
-
- _ValidateStartAndEndAgainstJSONStartAndEnd(
- start, end, bisect_contents['start'], bisect_contents['end'])
+ bisect_state = LoadStatusFile(args_output.last_tested, start, end)
+ if start != bisect_state['start'] or end != bisect_state['end']:
+ raise ValueError(f'The start {start} or the end {end} version provided is '
+ f'different than "start" {bisect_state["start"]} or "end" '
+ f'{bisect_state["end"]} in the .JSON file')
# Pending and skipped revisions are between 'start_revision' and
# 'end_revision'.
start_revision, end_revision, pending_revisions, skip_revisions = \
- GetStartAndEndRevision(start, end, bisect_contents['jobs'])
+ GetRemainingRange(start, end, bisect_state['jobs'])
- revisions, git_hashes = GetRevisionsListAndHashList(
- start_revision, end_revision, args_output.parallel, args_output.src_path,
- pending_revisions, skip_revisions)
+ revisions, git_hashes = GetCommitsBetween(start_revision, end_revision,
+ args_output.parallel,
+ args_output.src_path,
+ pending_revisions, skip_revisions)
# No more revisions between 'start_revision' and 'end_revision', so
# bisection is complete.
@@ -421,39 +325,61 @@ def main(args_output):
# This is determined by finding all valid revisions between 'start_revision'
# and 'end_revision' and that are NOT in the 'pending' and 'skipped' set.
if not revisions:
- # Successfully completed bisection where there are 2 cases:
- # 1) 'start_revision' and 'end_revision' are back-to-back (example:
- # 'start_revision' is 369410 and 'end_revision' is 369411).
- #
- # 2) 'start_revision' and 'end_revision' are NOT back-to-back, so there must
- # be tryjobs in between which are labeled as 'skip' for their 'status'
- # value.
- #
- # In either case, there are no 'pending' jobs.
- if not pending_revisions:
- _NoteCompletedBisection(args_output.last_tested, args_output.src_path,
- end_revision)
+ if pending_revisions:
+ # Some tryjobs are not finished which may change the actual bad
+ # commit/revision when those tryjobs are finished.
+ no_revisions_message = (f'No revisions between start {start_revision} '
+ f'and end {end_revision} to create tryjobs\n')
- if skip_revisions:
- skip_revisions_message = ('\nThe following revisions were skipped:\n' +
- '\n'.join(str(rev) for rev in skip_revisions))
+ if pending_revisions:
+ no_revisions_message += (
+ 'The following tryjobs are pending:\n' +
+ '\n'.join(str(rev) for rev in pending_revisions) + '\n')
- print(skip_revisions_message)
-
- return BisectionExitStatus.BISECTION_COMPLETE.value
-
- # Some tryjobs are not finished which may change the actual bad
- # commit/revision when those tryjobs are finished.
- DieWithNoRevisionsError(start_revision, end_revision, skip_revisions,
- pending_revisions)
-
- CheckForExistingTryjobsInRevisionsToLaunch(revisions, bisect_contents['jobs'])
+ if skip_revisions:
+ no_revisions_message += ('The following tryjobs were skipped:\n' +
+ '\n'.join(str(rev) for rev in skip_revisions) +
+ '\n')
+
+ raise ValueError(no_revisions_message)
+
+ print(f'Finished bisecting for {args_output.last_tested}')
+ if args_output.src_path:
+ bad_llvm_hash = get_llvm_hash.GetGitHashFrom(args_output.src_path,
+ end_revision)
+ else:
+ bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_revision)
+ print(f'The bad revision is {end_revision} and its commit hash is '
+ f'{bad_llvm_hash}')
+ if skip_revisions:
+ skip_revisions_message = ('\nThe following revisions were skipped:\n' +
+ '\n'.join(str(rev) for rev in skip_revisions))
+ print(skip_revisions_message)
+
+ if args_output.cleanup:
+ # Abondon all the CLs created for bisection
+ gerrit = os.path.join(args_output.chroot_path, 'chromite/bin/gerrit')
+ for build in bisect_state['jobs']:
+ try:
+ subprocess.check_output([gerrit, 'abandon', build['cl']],
+ stderr=subprocess.STDOUT,
+ encoding='utf-8')
+ except subprocess.CalledProcessError as err:
+ # the CL may have been abandoned
+ if 'chromite.lib.gob_util.GOBError' not in err.output:
+ raise
+
+ return BisectionExitStatus.BISECTION_COMPLETE.value
- UpdateBisection(revisions, git_hashes, bisect_contents,
- args_output.last_tested, update_packages,
- args_output.chroot_path, patch_metadata_file,
- args_output.extra_change_lists, args_output.options,
- args_output.builder, args_output.verbose)
+ for rev in revisions:
+ if update_tryjob_status.FindTryjobIndex(rev,
+ bisect_state['jobs']) is not None:
+ raise ValueError(f'Revision {rev} exists already in "jobs"')
+
+ Bisect(revisions, git_hashes, bisect_state, args_output.last_tested,
+ update_packages, args_output.chroot_path, patch_metadata_file,
+ args_output.extra_change_lists, args_output.options,
+ args_output.builder, args_output.verbose)
if __name__ == '__main__':
diff --git a/llvm_tools/llvm_bisection_unittest.py b/llvm_tools/llvm_bisection_unittest.py
index e730293b..a40770a5 100755
--- a/llvm_tools/llvm_bisection_unittest.py
+++ b/llvm_tools/llvm_bisection_unittest.py
@@ -11,50 +11,57 @@
from __future__ import print_function
import json
+import os
+import subprocess
import unittest
import unittest.mock as mock
import chroot
import get_llvm_hash
+import git_llvm_rev
import llvm_bisection
import modify_a_tryjob
import test_helpers
-import update_tryjob_status
class LLVMBisectionTest(unittest.TestCase):
"""Unittests for LLVM bisection."""
- def testStartAndEndDoNotMatchJsonStartAndEnd(self):
+ def testGetRemainingRangePassed(self):
start = 100
end = 150
- json_start = 110
- json_end = 150
-
- # Verify the exception is raised when the start and end revision for LLVM
- # bisection do not match the .JSON's 'start' and 'end' values.
- with self.assertRaises(ValueError) as err:
- llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd(
- start, end, json_start, json_end)
-
- expected_error_message = ('The start %d or the end %d version provided is '
- 'different than "start" %d or "end" %d in the '
- '.JSON file' % (start, end, json_start, json_end))
-
- self.assertEqual(str(err.exception), expected_error_message)
-
- def testStartAndEndMatchJsonStartAndEnd(self):
- start = 100
- end = 150
+ test_tryjobs = [{
+ 'rev': 110,
+ 'status': 'good',
+ 'link': 'https://some_tryjob_1_url.com'
+ }, {
+ 'rev': 120,
+ 'status': 'good',
+ 'link': 'https://some_tryjob_2_url.com'
+ }, {
+ 'rev': 130,
+ 'status': 'pending',
+ 'link': 'https://some_tryjob_3_url.com'
+ }, {
+ 'rev': 135,
+ 'status': 'skip',
+ 'link': 'https://some_tryjob_4_url.com'
+ }, {
+ 'rev': 140,
+ 'status': 'bad',
+ 'link': 'https://some_tryjob_5_url.com'
+ }]
- json_start = 100
- json_end = 150
+ # Tuple consists of the new good revision, the new bad revision, a set of
+ # 'pending' revisions, and a set of 'skip' revisions.
+ expected_revisions_tuple = 120, 140, {130}, {135}
- llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd(
- start, end, json_start, json_end)
+ self.assertEqual(
+ llvm_bisection.GetRemainingRange(start, end, test_tryjobs),
+ expected_revisions_tuple)
- def testTryjobStatusIsMissing(self):
+ def testGetRemainingRangeFailedWithMissingStatus(self):
start = 100
end = 150
@@ -72,18 +79,14 @@ class LLVMBisectionTest(unittest.TestCase):
'link': 'https://some_tryjob_3_url.com'
}]
- # Verify the exception is raised when a tryjob does not have a value for
- # the 'status' key or the 'status' key is missing.
with self.assertRaises(ValueError) as err:
- llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs)
+ llvm_bisection.GetRemainingRange(start, end, test_tryjobs)
- expected_error_message = (
- '"status" is missing or has no value, please '
- 'go to %s and update it' % test_tryjobs[1]['link'])
+ error_message = ('"status" is missing or has no value, please '
+ 'go to %s and update it' % test_tryjobs[1]['link'])
+ self.assertEqual(str(err.exception), error_message)
- self.assertEqual(str(err.exception), expected_error_message)
-
- def testGoodRevisionGreaterThanBadRevision(self):
+ def testGetRemainingRangeFailedWithInvalidRange(self):
start = 100
end = 150
@@ -101,206 +104,68 @@ class LLVMBisectionTest(unittest.TestCase):
'link': 'https://some_tryjob_3_url.com'
}]
- # Verify the exception is raised when the new 'start' revision is greater
- # than the new 'bad' revision for bisection (i.e. bisection is broken).
with self.assertRaises(AssertionError) as err:
- llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs)
+ llvm_bisection.GetRemainingRange(start, end, test_tryjobs)
- expected_error_message = (
- 'Bisection is broken because %d (good) is >= '
- '%d (bad)' % (test_tryjobs[2]['rev'], test_tryjobs[0]['rev']))
+ expected_error_message = ('Bisection is broken because %d (good) is >= '
+ '%d (bad)' %
+ (test_tryjobs[2]['rev'], test_tryjobs[0]['rev']))
self.assertEqual(str(err.exception), expected_error_message)
- def testSuccessfullyGetNewStartAndNewEndRevision(self):
- start = 100
- end = 150
-
- test_tryjobs = [{
- 'rev': 110,
- 'status': 'good',
- 'link': 'https://some_tryjob_1_url.com'
- }, {
- 'rev': 120,
- 'status': 'good',
- 'link': 'https://some_tryjob_2_url.com'
- }, {
- 'rev': 130,
- 'status': 'pending',
- 'link': 'https://some_tryjob_3_url.com'
- }, {
- 'rev': 135,
- 'status': 'skip',
- 'link': 'https://some_tryjob_4_url.com'
- }, {
- 'rev': 140,
- 'status': 'bad',
- 'link': 'https://some_tryjob_5_url.com'
- }]
-
- # Tuple consists of the new good revision, the new bad revision, a set of
- # 'pending' revisions, and a set of 'skip' revisions.
- expected_revisions_tuple = 120, 140, {130}, {135}
-
- self.assertTupleEqual(
- llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs),
- expected_revisions_tuple)
-
@mock.patch.object(get_llvm_hash, 'GetGitHashFrom')
- def testNoRevisionsBetweenStartAndEnd(self, mock_get_git_hash):
- start = 100
- end = 110
-
- test_pending_revisions = {107}
- test_skip_revisions = {101, 102, 103, 104, 108, 109}
-
- # Simulate behavior of `GetGitHashFrom()` when the revision does not
- # exist in the LLVM source tree.
- def MockGetGitHashForRevisionRaiseException(_src_path, _revision):
- raise ValueError('Revision does not exist')
-
- mock_get_git_hash.side_effect = MockGetGitHashForRevisionRaiseException
-
- parallel = 3
-
- abs_path_to_src = '/abs/path/to/src'
-
- self.assertListEqual(
- llvm_bisection.GetRevisionsBetweenBisection(
- start, end, parallel, abs_path_to_src, test_pending_revisions,
- test_skip_revisions), [])
-
- # Assume llvm_bisection module has imported GetGitHashFrom
- @mock.patch.object(get_llvm_hash, 'GetGitHashFrom')
- def testSuccessfullyRetrievedRevisionsBetweenStartAndEnd(
- self, mock_get_git_hash):
-
- start = 100
- end = 110
-
- test_pending_revisions = set()
- test_skip_revisions = {101, 102, 103, 104, 106, 108, 109}
-
+ def testGetCommitsBetweenPassed(self, mock_get_git_hash):
+ start = git_llvm_rev.base_llvm_revision
+ end = start + 10
+ test_pending_revisions = {start + 7}
+ test_skip_revisions = {
+ start + 1, start + 2, start + 4, start + 8, start + 9
+ }
parallel = 3
-
abs_path_to_src = '/abs/path/to/src'
- # Valid revision that exist in the LLVM source tree between 'start' and
- # 'end' and were not in the 'pending' set or 'skip' set.
- expected_revisions_between_start_and_end = [105, 107]
-
- self.assertListEqual(
- llvm_bisection.GetRevisionsBetweenBisection(
- start, end, parallel, abs_path_to_src, test_pending_revisions,
- test_skip_revisions), expected_revisions_between_start_and_end)
-
- self.assertEqual(mock_get_git_hash.call_count, 2)
+ revs = ['a123testhash3', 'a123testhash5']
+ mock_get_git_hash.side_effect = revs
- # Simulate behavior of `GetGitHashFrom()` when successfully retrieved
- # a list git hashes for each revision in the revisions list.
- # Assume llvm_bisection module has imported GetGitHashFrom
- @mock.patch.object(get_llvm_hash, 'GetGitHashFrom')
- # Simulate behavior of `GetRevisionsBetweenBisection()` when successfully
- # retrieved a list of valid revisions between 'start' and 'end'.
- @mock.patch.object(llvm_bisection, 'GetRevisionsBetweenBisection')
- # Simulate behavior of `CreatTempLLVMRepo()` when successfully created a
- # worktree when a source path was not provided.
- @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo')
- def testSuccessfullyGetRevisionsListAndHashList(
- self, mock_create_temp_llvm_repo, mock_get_revisions_between_bisection,
- mock_get_git_hash):
-
- expected_revisions_and_hash_tuple = ([102, 105, 108], [
- 'a123testhash1', 'a123testhash2', 'a123testhash3'
- ])
-
- @test_helpers.CallCountsToMockFunctions
- def MockGetGitHashForRevision(call_count, _src_path, _rev):
- # Simulate retrieving the git hash for the revision.
- if call_count < 3:
- return expected_revisions_and_hash_tuple[1][call_count]
-
- assert False, 'Called `GetGitHashFrom()` more than expected.'
-
- temp_worktree = '/abs/path/to/tmpDir'
-
- mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = \
- temp_worktree
-
- # Simulate the valid revisions list.
- mock_get_revisions_between_bisection.return_value = \
- expected_revisions_and_hash_tuple[0]
-
- # Simulate behavior of `GetGitHashFrom()` by using the testing
- # function.
- mock_get_git_hash.side_effect = MockGetGitHashForRevision
+ git_hashes = [
+ git_llvm_rev.base_llvm_revision + 3, git_llvm_rev.base_llvm_revision + 5
+ ]
- start = 100
- end = 110
- parallel = 3
- src_path = None
- pending_revisions = {103, 104}
- skip_revisions = {101, 106, 107, 109}
-
- self.assertTupleEqual(
- llvm_bisection.GetRevisionsListAndHashList(
- start, end, parallel, src_path, pending_revisions, skip_revisions),
- expected_revisions_and_hash_tuple)
-
- mock_get_revisions_between_bisection.assert_called_once()
-
- self.assertEqual(mock_get_git_hash.call_count, 3)
+ self.assertEqual(
+ llvm_bisection.GetCommitsBetween(start, end, parallel, abs_path_to_src,
+ test_pending_revisions,
+ test_skip_revisions),
+ (git_hashes, revs))
- def testSuccessfullyDieWithNoRevisionsError(self):
+ def testLoadStatusFilePassedWithExistingFile(self):
start = 100
- end = 110
-
- pending_revisions = {105, 108}
- skip_revisions = {101, 102, 103, 104, 106, 107, 109}
-
- expected_no_revisions_message = ('No revisions between start %d and end '
- '%d to create tryjobs' % (start, end))
-
- expected_no_revisions_message += '\nThe following tryjobs are pending:\n' \
- + '\n'.join(str(rev) for rev in pending_revisions)
-
- expected_no_revisions_message += '\nThe following tryjobs were skipped:\n' \
- + '\n'.join(str(rev) for rev in skip_revisions)
-
- # Verify that an exception is raised when there are no revisions to launch
- # tryjobs for between 'start' and 'end' and some tryjobs are 'pending'.
- with self.assertRaises(ValueError) as err:
- llvm_bisection.DieWithNoRevisionsError(start, end, skip_revisions,
- pending_revisions)
+ end = 150
- self.assertEqual(str(err.exception), expected_no_revisions_message)
+ test_bisect_state = {'start': start, 'end': end, 'jobs': []}
- # Simulate behavior of `FindTryjobIndex()` when the index of the tryjob was
- # found.
- @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
- def testTryjobExistsInRevisionsToLaunch(self, mock_find_tryjob_index):
- test_existing_jobs = [{'rev': 102, 'status': 'good'}]
+ # Simulate that the status file exists.
+ with test_helpers.CreateTemporaryJsonFile() as temp_json_file:
+ with open(temp_json_file, 'w') as f:
+ test_helpers.WritePrettyJsonFile(test_bisect_state, f)
- revision_to_launch = [102]
+ self.assertEqual(
+ llvm_bisection.LoadStatusFile(temp_json_file, start, end),
+ test_bisect_state)
- expected_revision_that_exists = 102
+ def testLoadStatusFilePassedWithoutExistingFile(self):
+ start = 200
+ end = 250
- with self.assertRaises(ValueError) as err:
- llvm_bisection.CheckForExistingTryjobsInRevisionsToLaunch(
- revision_to_launch, test_existing_jobs)
+ expected_bisect_state = {'start': start, 'end': end, 'jobs': []}
- expected_found_tryjob_index_error_message = (
- 'Revision %d exists already '
- 'in "jobs"' % expected_revision_that_exists)
+ last_tested = '/abs/path/to/file_that_does_not_exist.json'
self.assertEqual(
- str(err.exception), expected_found_tryjob_index_error_message)
-
- mock_find_tryjob_index.assert_called_once()
+ llvm_bisection.LoadStatusFile(last_tested, start, end),
+ expected_bisect_state)
@mock.patch.object(modify_a_tryjob, 'AddTryjob')
- def testSuccessfullyUpdatedStatusFileWhenExceptionIsRaised(
- self, mock_add_tryjob):
+ def testBisectPassed(self, mock_add_tryjob):
git_hash_list = ['a123testhash1', 'a123testhash2', 'a123testhash3']
revisions_list = [102, 104, 106]
@@ -343,11 +208,11 @@ class LLVMBisectionTest(unittest.TestCase):
# Verify that the status file is updated when an exception happened when
# attempting to launch a revision (i.e. progress is not lost).
with self.assertRaises(ValueError) as err:
- llvm_bisection.UpdateBisection(
- revisions_list, git_hash_list, bisection_contents, temp_json_file,
- packages, args_output.chroot_path, patch_file,
- args_output.extra_change_lists, args_output.options,
- args_output.builders, args_output.verbose)
+ llvm_bisection.Bisect(revisions_list, git_hash_list, bisection_contents,
+ temp_json_file, packages, args_output.chroot_path,
+ patch_file, args_output.extra_change_lists,
+ args_output.options, args_output.builders,
+ args_output.verbose)
expected_bisection_contents = {
'start':
@@ -368,121 +233,143 @@ class LLVMBisectionTest(unittest.TestCase):
with open(temp_json_file) as f:
json_contents = json.load(f)
- self.assertDictEqual(json_contents, expected_bisection_contents)
+ self.assertEqual(json_contents, expected_bisection_contents)
self.assertEqual(str(err.exception), 'Unable to launch tryjob')
self.assertEqual(mock_add_tryjob.call_count, 3)
- # Simulate behavior of `GetGitHashFrom()` when successfully retrieved
- # the git hash of the bad revision. Assume llvm_bisection has imported
- # GetGitHashFrom
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
@mock.patch.object(
- get_llvm_hash, 'GetGitHashFrom', return_value='a123testhash4')
- def testCompletedBisectionWhenProvidedSrcPath(self, mock_get_git_hash):
- last_tested = '/some/last/tested_file.json'
+ get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4')
+ @mock.patch.object(llvm_bisection, 'GetCommitsBetween')
+ @mock.patch.object(llvm_bisection, 'GetRemainingRange')
+ @mock.patch.object(llvm_bisection, 'LoadStatusFile')
+ @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
+ def testMainPassed(self, mock_outside_chroot, mock_load_status_file,
+ mock_get_range, mock_get_revision_and_hash_list,
+ _mock_get_bad_llvm_hash, mock_abandon_cl):
- src_path = '/abs/path/to/src/path'
+ start = 500
+ end = 502
+ cl = 1
- # The bad revision.
- end = 150
+ bisect_state = {
+ 'start': start,
+ 'end': end,
+ 'jobs': [{
+ 'rev': 501,
+ 'status': 'bad',
+ 'cl': cl
+ }]
+ }
- llvm_bisection._NoteCompletedBisection(last_tested, src_path, end)
+ skip_revisions = {501}
+ pending_revisions = {}
- mock_get_git_hash.assert_called_once()
+ mock_load_status_file.return_value = bisect_state
- # Simulate behavior of `GetLLVMHash()` when successfully retrieved
- # the git hash of the bad revision.
- @mock.patch.object(
- get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash5')
- def testCompletedBisectionWhenNotProvidedSrcPath(self, mock_get_git_hash):
- last_tested = '/some/last/tested_file.json'
+ mock_get_range.return_value = (start, end, pending_revisions,
+ skip_revisions)
- src_path = None
+ mock_get_revision_and_hash_list.return_value = [], []
- # The bad revision.
- end = 200
+ args_output = test_helpers.ArgsOutputTest()
+ args_output.start_rev = start
+ args_output.end_rev = end
+ args_output.parallel = 3
+ args_output.src_path = None
+ args_output.chroot_path = 'somepath'
+ args_output.cleanup = True
- llvm_bisection._NoteCompletedBisection(last_tested, src_path, end)
+ self.assertEqual(
+ llvm_bisection.main(args_output),
+ llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value)
- mock_get_git_hash.assert_called_once()
+ mock_outside_chroot.assert_called_once()
- def testSuccessfullyLoadedStatusFile(self):
- start = 100
- end = 150
+ mock_load_status_file.assert_called_once()
- test_bisect_contents = {'start': start, 'end': end, 'jobs': []}
+ mock_get_range.assert_called_once()
- # Simulate that the status file exists.
- with test_helpers.CreateTemporaryJsonFile() as temp_json_file:
- with open(temp_json_file, 'w') as f:
- test_helpers.WritePrettyJsonFile(test_bisect_contents, f)
+ mock_get_revision_and_hash_list.assert_called_once()
- self.assertDictEqual(
- llvm_bisection.LoadStatusFile(temp_json_file, start, end),
- test_bisect_contents)
+ mock_abandon_cl.assert_called_once()
+ self.assertEqual(
+ mock_abandon_cl.call_args,
+ mock.call(
+ [
+ os.path.join(args_output.chroot_path, 'chromite/bin/gerrit'),
+ 'abandon',
+ cl,
+ ],
+ stderr=subprocess.STDOUT,
+ encoding='utf-8',
+ ))
- def testLoadedStatusFileThatDoesNotExist(self):
- start = 200
- end = 250
+ @mock.patch.object(llvm_bisection, 'LoadStatusFile')
+ @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
+ def testMainFailedWithInvalidRange(self, mock_outside_chroot,
+ mock_load_status_file):
- expected_bisect_contents = {'start': start, 'end': end, 'jobs': []}
+ start = 500
+ end = 502
- last_tested = '/abs/path/to/file_that_does_not_exist.json'
+ bisect_state = {
+ 'start': start - 1,
+ 'end': end,
+ }
- self.assertDictEqual(
- llvm_bisection.LoadStatusFile(last_tested, start, end),
- expected_bisect_contents)
-
- # Simulate behavior of `_NoteCompletedBisection()` when there are no more
- # tryjobs to launch between start and end, so bisection is complete.
- @mock.patch.object(llvm_bisection, '_NoteCompletedBisection')
- @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList')
- @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision')
- # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when
- # both start and end revisions match.
- @mock.patch.object(llvm_bisection,
- '_ValidateStartAndEndAgainstJSONStartAndEnd')
+ mock_load_status_file.return_value = bisect_state
+
+ args_output = test_helpers.ArgsOutputTest()
+ args_output.start_rev = start
+ args_output.end_rev = end
+ args_output.parallel = 3
+ args_output.src_path = None
+
+ with self.assertRaises(ValueError) as err:
+ llvm_bisection.main(args_output)
+
+ error_message = (f'The start {start} or the end {end} version provided is '
+ f'different than "start" {bisect_state["start"]} or "end" '
+ f'{bisect_state["end"]} in the .JSON file')
+
+ self.assertEqual(str(err.exception), error_message)
+
+ mock_outside_chroot.assert_called_once()
+
+ mock_load_status_file.assert_called_once()
+
+ @mock.patch.object(llvm_bisection, 'GetCommitsBetween')
+ @mock.patch.object(llvm_bisection, 'GetRemainingRange')
@mock.patch.object(llvm_bisection, 'LoadStatusFile')
- # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the
- # script outside of the chroot.
@mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
- def testSuccessfullyBisectedLLVM(
- self, mock_outside_chroot, mock_load_status_file,
- mock_validate_start_and_end, mock_get_start_and_end_revision,
- mock_get_revision_and_hash_list, mock_note_completed_bisection):
+ def testMainFailedWithPendingBuilds(self, mock_outside_chroot,
+ mock_load_status_file, mock_get_range,
+ mock_get_revision_and_hash_list):
start = 500
end = 502
+ rev = 501
- bisect_contents = {
+ bisect_state = {
'start': start,
'end': end,
'jobs': [{
- 'rev': 501,
- 'status': 'skip'
+ 'rev': rev,
+ 'status': 'pending'
}]
}
- skip_revisions = {501}
- pending_revisions = {}
+ skip_revisions = {}
+ pending_revisions = {rev}
+
+ mock_load_status_file.return_value = bisect_state
+
+ mock_get_range.return_value = (start, end, pending_revisions,
+ skip_revisions)
- # Simulate behavior of `LoadStatusFile()` when successfully loaded the
- # status file.
- mock_load_status_file.return_value = bisect_contents
-
- # Simulate behavior of `GetStartAndEndRevision()` when successfully found
- # the new start and end revision of the bisection.
- #
- # Returns new start revision, new end revision, a set of pending revisions,
- # and a set of skip revisions.
- mock_get_start_and_end_revision.return_value = (start, end,
- pending_revisions,
- skip_revisions)
-
- # Simulate behavior of `GetRevisionsListAndHashList()` when successfully
- # retrieved valid revisions (along with their git hashes) between start and
- # end (in this case, none).
mock_get_revision_and_hash_list.return_value = [], []
args_output = test_helpers.ArgsOutputTest()
@@ -491,110 +378,131 @@ class LLVMBisectionTest(unittest.TestCase):
args_output.parallel = 3
args_output.src_path = None
- self.assertEqual(
- llvm_bisection.main(args_output),
- llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value)
+ with self.assertRaises(ValueError) as err:
+ llvm_bisection.main(args_output)
+
+ error_message = (f'No revisions between start {start} and end {end} to '
+ 'create tryjobs\nThe following tryjobs are pending:\n'
+ f'{rev}\n')
+
+ self.assertEqual(str(err.exception), error_message)
mock_outside_chroot.assert_called_once()
mock_load_status_file.assert_called_once()
- mock_validate_start_and_end.assert_called_once()
-
- mock_get_start_and_end_revision.assert_called_once()
+ mock_get_range.assert_called_once()
mock_get_revision_and_hash_list.assert_called_once()
- mock_note_completed_bisection.assert_called_once()
-
- @mock.patch.object(llvm_bisection, 'DieWithNoRevisionsError')
- # Simulate behavior of `_NoteCompletedBisection()` when there are no more
- # tryjobs to launch between start and end, so bisection is complete.
- @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList')
- @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision')
- # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when
- # both start and end revisions match.
- @mock.patch.object(llvm_bisection,
- '_ValidateStartAndEndAgainstJSONStartAndEnd')
+ @mock.patch.object(llvm_bisection, 'GetCommitsBetween')
+ @mock.patch.object(llvm_bisection, 'GetRemainingRange')
@mock.patch.object(llvm_bisection, 'LoadStatusFile')
- # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the
- # script outside of the chroot.
@mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
- def testNoMoreTryjobsToLaunch(
- self, mock_outside_chroot, mock_load_status_file,
- mock_validate_start_and_end, mock_get_start_and_end_revision,
- mock_get_revision_and_hash_list, mock_die_with_no_revisions_error):
+ def testMainFailedWithDuplicateBuilds(self, mock_outside_chroot,
+ mock_load_status_file, mock_get_range,
+ mock_get_revision_and_hash_list):
start = 500
end = 502
+ rev = 501
+ git_hash = 'a123testhash1'
- bisect_contents = {
+ bisect_state = {
'start': start,
'end': end,
'jobs': [{
- 'rev': 501,
+ 'rev': rev,
'status': 'pending'
}]
}
skip_revisions = {}
- pending_revisions = {501}
-
- no_revisions_error_message = ('No more tryjobs to launch between %d and '
- '%d' % (start, end))
-
- def MockNoRevisionsErrorException(_start, _end, _skip, _pending):
- raise ValueError(no_revisions_error_message)
-
- # Simulate behavior of `LoadStatusFile()` when successfully loaded the
- # status file.
- mock_load_status_file.return_value = bisect_contents
-
- # Simulate behavior of `GetStartAndEndRevision()` when successfully found
- # the new start and end revision of the bisection.
- #
- # Returns new start revision, new end revision, a set of pending revisions,
- # and a set of skip revisions.
- mock_get_start_and_end_revision.return_value = (start, end,
- pending_revisions,
- skip_revisions)
-
- # Simulate behavior of `GetRevisionsListAndHashList()` when successfully
- # retrieved valid revisions (along with their git hashes) between start and
- # end (in this case, none).
- mock_get_revision_and_hash_list.return_value = [], []
+ pending_revisions = {rev}
+
+ mock_load_status_file.return_value = bisect_state
- # Use the test function to simulate `DieWithNoRevisionsWithError()`
- # behavior.
- mock_die_with_no_revisions_error.side_effect = MockNoRevisionsErrorException
+ mock_get_range.return_value = (start, end, pending_revisions,
+ skip_revisions)
+
+ mock_get_revision_and_hash_list.return_value = [rev], [git_hash]
- # Simulate behavior of arguments passed into the command line.
args_output = test_helpers.ArgsOutputTest()
args_output.start_rev = start
args_output.end_rev = end
args_output.parallel = 3
args_output.src_path = None
- # Verify the exception is raised when there are no more tryjobs to launch
- # between start and end when there are tryjobs that are 'pending', so
- # the actual bad revision can change when those tryjobs's 'status' are
- # updated.
with self.assertRaises(ValueError) as err:
llvm_bisection.main(args_output)
- self.assertEqual(str(err.exception), no_revisions_error_message)
+ error_message = ('Revision %d exists already in "jobs"' % rev)
+ self.assertEqual(str(err.exception), error_message)
mock_outside_chroot.assert_called_once()
mock_load_status_file.assert_called_once()
- mock_validate_start_and_end.assert_called_once()
-
- mock_get_start_and_end_revision.assert_called_once()
+ mock_get_range.assert_called_once()
mock_get_revision_and_hash_list.assert_called_once()
- mock_die_with_no_revisions_error.assert_called_once()
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
+ @mock.patch.object(
+ get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4')
+ @mock.patch.object(llvm_bisection, 'GetCommitsBetween')
+ @mock.patch.object(llvm_bisection, 'GetRemainingRange')
+ @mock.patch.object(llvm_bisection, 'LoadStatusFile')
+ @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
+ def testMainFailedToAbandonCL(self, mock_outside_chroot,
+ mock_load_status_file, mock_get_range,
+ mock_get_revision_and_hash_list,
+ _mock_get_bad_llvm_hash, mock_abandon_cl):
+
+ start = 500
+ end = 502
+
+ bisect_state = {
+ 'start': start,
+ 'end': end,
+ 'jobs': [{
+ 'rev': 501,
+ 'status': 'bad',
+ 'cl': 0
+ }]
+ }
+
+ skip_revisions = {501}
+ pending_revisions = {}
+
+ mock_load_status_file.return_value = bisect_state
+
+ mock_get_range.return_value = (start, end, pending_revisions,
+ skip_revisions)
+
+ mock_get_revision_and_hash_list.return_value = ([], [])
+
+ error_message = 'Error message.'
+ mock_abandon_cl.side_effect = subprocess.CalledProcessError(
+ returncode=1, cmd=[], output=error_message)
+
+ args_output = test_helpers.ArgsOutputTest()
+ args_output.start_rev = start
+ args_output.end_rev = end
+ args_output.parallel = 3
+ args_output.src_path = None
+ args_output.cleanup = True
+
+ with self.assertRaises(subprocess.CalledProcessError) as err:
+ llvm_bisection.main(args_output)
+
+ self.assertEqual(err.exception.output, error_message)
+
+ mock_outside_chroot.assert_called_once()
+
+ mock_load_status_file.assert_called_once()
+
+ mock_get_range.assert_called_once()
if __name__ == '__main__':
diff --git a/llvm_tools/llvm_project.py b/llvm_tools/llvm_project.py
index c171370c..7937729f 100644
--- a/llvm_tools/llvm_project.py
+++ b/llvm_tools/llvm_project.py
@@ -13,6 +13,7 @@ import subprocess
import sys
import get_llvm_hash
+import git_llvm_rev
def get_location() -> str:
@@ -33,9 +34,13 @@ def ensure_up_to_date():
assert checkout == actual_checkout, '%s != %s' % (actual_checkout, checkout)
commit_timestamp = subprocess.check_output(
- ['git', 'log', '-n1', '--format=%ct', 'origin/master'],
+ [
+ 'git', 'log', '-n1', '--format=%ct',
+ 'origin/' + git_llvm_rev.MAIN_BRANCH
+ ],
cwd=checkout,
- encoding='utf-8')
+ encoding='utf-8',
+ )
commit_time = datetime.datetime.fromtimestamp(int(commit_timestamp.strip()))
now = datetime.datetime.now()
@@ -52,7 +57,7 @@ def ensure_up_to_date():
(time_since_last_commit.days, checkout),
file=sys.stderr)
- result = subprocess.run(['git', 'fetch', 'origin'], cwd=checkout)
+ result = subprocess.run(['git', 'fetch', 'origin'], check=False, cwd=checkout)
if result.returncode:
print(
'Sync failed somehow; hoping that things are fresh enough, then...',
diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py
index 71b5574b..3a23890a 100755
--- a/llvm_tools/nightly_revert_checker.py
+++ b/llvm_tools/nightly_revert_checker.py
@@ -27,6 +27,7 @@ import typing as t
import cros_utils.email_sender as email_sender
import cros_utils.tiny_render as tiny_render
import get_llvm_hash
+import git_llvm_rev
import revert_checker
State = t.Any
@@ -44,12 +45,12 @@ def _find_interesting_android_shas(
encoding='utf-8',
).strip()
- master_legacy = get_llvm_merge_base('aosp/master-legacy')
+ main_legacy = get_llvm_merge_base('aosp/master-legacy')
testing_upstream = get_llvm_merge_base('aosp/testing-upstream')
- result = [('master-legacy', master_legacy)]
+ result = [('main-legacy', main_legacy)]
# If these are the same SHA, there's no point in tracking both.
- if master_legacy != testing_upstream:
+ if main_legacy != testing_upstream:
result.append(('testing-upstream', testing_upstream))
return result
@@ -270,7 +271,7 @@ def main(argv: t.List[str]) -> None:
for friendly_name, sha in interesting_shas:
logging.info('Finding reverts across %s (%s)', friendly_name, sha)
all_reverts = revert_checker.find_reverts(
- llvm_dir, sha, root='origin/master')
+ llvm_dir, sha, root='origin/' + git_llvm_rev.MAIN_BRANCH)
logging.info('Detected the following revert(s) across %s:\n%s',
friendly_name, pprint.pformat(all_reverts))
diff --git a/llvm_tools/upload_lexan_crashes_to_forcey.py b/llvm_tools/upload_lexan_crashes_to_forcey.py
new file mode 100755
index 00000000..b93f51a7
--- /dev/null
+++ b/llvm_tools/upload_lexan_crashes_to_forcey.py
@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches and submits the latest test-cases from Lexan's crash bucket."""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import contextlib
+import datetime
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Generator, List, Iterable
+
+gsurl_base = 'gs://chrome-clang-crash-reports/v1'
+
+
+def gsutil_ls(loc: str) -> List[str]:
+ results = subprocess.run(['gsutil.py', 'ls', loc],
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding='utf-8')
+ return [l.strip() for l in results.stdout.splitlines()]
+
+
+def gsurl_ls_last_numbers(url: str) -> List[int]:
+ return sorted(int(x.rstrip('/').split('/')[-1]) for x in gsutil_ls(url))
+
+
+def get_available_year_numbers() -> List[int]:
+ return gsurl_ls_last_numbers(gsurl_base)
+
+
+def get_available_month_numbers(year: int) -> List[int]:
+ return gsurl_ls_last_numbers(f'{gsurl_base}/{year}')
+
+
+def get_available_day_numbers(year: int, month: int) -> List[int]:
+ return gsurl_ls_last_numbers(f'{gsurl_base}/{year}/{month:02d}')
+
+
+def get_available_test_case_urls(year: int, month: int, day: int) -> List[str]:
+ return gsutil_ls(f'{gsurl_base}/{year}/{month:02d}/{day:02d}')
+
+
+def test_cases_on_or_after(date: datetime.datetime
+ ) -> Generator[str, None, None]:
+ """Yields all test-cases submitted on or after the given date."""
+ for year in get_available_year_numbers():
+ if year < date.year:
+ continue
+
+ for month in get_available_month_numbers(year):
+ if year == date.year and month < date.month:
+ continue
+
+ for day in get_available_day_numbers(year, month):
+ when = datetime.date(year, month, day)
+ if when < date:
+ continue
+
+ yield when, get_available_test_case_urls(year, month, day)
+
+
+def to_ymd(date: datetime.date) -> str:
+ return date.strftime('%Y-%m-%d')
+
+
+def from_ymd(date_str: str) -> datetime.date:
+ return datetime.datetime.strptime(date_str, '%Y-%m-%d').date()
+
+
+def persist_state(seen_urls: Iterable[str], state_file: str,
+ current_date: datetime.date):
+ tmp_state_file = state_file + '.tmp'
+ with open(tmp_state_file, 'w', encoding='utf-8') as f:
+ json.dump(
+ {
+ 'already_seen': sorted(seen_urls),
+ 'most_recent_date': to_ymd(current_date),
+ },
+ f,
+ )
+ os.rename(tmp_state_file, state_file)
+
+
+@contextlib.contextmanager
+def temp_dir() -> Generator[str, None, None]:
+ loc = tempfile.mkdtemp('lexan-autosubmit')
+ try:
+ yield loc
+ finally:
+ shutil.rmtree(loc)
+
+
+def submit_test_case(gs_url: str, cr_tool: str) -> None:
+ logging.info('Submitting %s', gs_url)
+ suffix = os.path.splitext(gs_url)[1]
+ with temp_dir() as tempdir:
+ target_name = 'test_case' + suffix
+ target = os.path.join(tempdir, target_name)
+ subprocess.run(['gsutil.py', 'cp', gs_url, target], check=True)
+ subprocess.run(['tar', 'xaf', target_name], check=True, cwd=tempdir)
+ os.unlink(target)
+
+ # Sometimes (e.g., in
+ # gs://chrome-clang-crash-reports/v1/2020/03/27/
+ # chromium.clang-ToTiOS-12754-GTXToolKit-2bfcde.tgz)
+ # we'll get `.crash` files. Unclear why, but let's filter them out anyway.
+ repro_files = [x for x in os.listdir(tempdir) if not x.endswith('.crash')]
+ assert len(repro_files) == 2, repro_files
+ if repro_files[0].endswith('.sh'):
+ sh_file, src_file = repro_files
+ assert not src_file.endswith('.sh'), repro_files
+ else:
+ src_file, sh_file = repro_files
+ assert sh_file.endswith('.sh'), repro_files
+
+ subprocess.run(
+ [
+ cr_tool,
+ 'reduce',
+ '-stream=false',
+ '-wait=false',
+ '-note',
+ gs_url,
+ '-sh_file',
+ os.path.join(tempdir, sh_file),
+ '-src_file',
+ os.path.join(tempdir, src_file),
+ ],
+ check=True,
+ )
+
+
+def submit_new_test_cases(
+ last_seen_test_cases: Iterable[str],
+ earliest_date_to_check: datetime.date,
+ forcey: str,
+ state_file_path: str,
+) -> None:
+ """Submits new test-cases to forcey.
+
+ This will persist state after each test-case is submitted.
+
+ Args:
+ last_seen_test_cases: test-cases which have been submitted already, and
+ should be skipped if seen again.
+ earliest_date_to_check: the earliest date we should consider test-cases
+ from.
+ forcey: path to the forcey binary.
+ state_file_path: path to our state file.
+ """
+ # `all_test_cases_seen` is the union of all test-cases seen on this and prior
+ # invocations. It guarantees, in all cases we care about, that we won't
+ # submit the same test-case twice. `test_cases_seen_this_invocation` is
+ # persisted as "all of the test-cases we've seen on this and prior
+ # invocations" if we successfully submit _all_ test-cases.
+ #
+ # Since you can visualize the test-cases this script considers as a sliding
+ # window that only moves forward, if we saw a test-case on a prior iteration
+ # but no longer see it, we'll never see it again (since it fell out of our
+ # sliding window by being too old). Hence, keeping it around is
+ # pointless.
+ #
+ # We only persist this minimized set of test-cases if _everything_ succeeds,
+ # since if something fails below, there's a chance that we haven't revisited
+ # test-cases that we've already seen.
+ all_test_cases_seen = set(last_seen_test_cases)
+ test_cases_seen_this_invocation = []
+ most_recent_date = earliest_date_to_check
+ for date, candidates in test_cases_on_or_after(earliest_date_to_check):
+ most_recent_date = max(most_recent_date, date)
+
+ for url in candidates:
+ test_cases_seen_this_invocation.append(url)
+ if url in all_test_cases_seen:
+ continue
+
+ all_test_cases_seen.add(url)
+ submit_test_case(url, forcey)
+
+ # Persisting on each iteration of this loop isn't free, but it's the
+ # easiest way to not resubmit test-cases, and it's good to keep in mind
+ # that:
+ # - the state file will be small (<12KB, since it only keeps a few days
+ # worth of test-cases after the first run)
+ # - in addition to this, we're downloading+unzipping+reuploading multiple
+ # MB of test-case bytes.
+ #
+ # So comparatively, the overhead here probably isn't an issue.
+ persist_state(all_test_cases_seen, state_file_path, most_recent_date)
+
+ persist_state(test_cases_seen_this_invocation, state_file_path,
+ most_recent_date)
+
+
+def main(argv: List[str]):
+ logging.basicConfig(
+ format='>> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: '
+ '%(message)s',
+ level=logging.INFO,
+ )
+
+ my_dir = os.path.dirname(os.path.abspath(__file__))
+
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ '--state_file', default=os.path.join(my_dir, 'lexan-state.json'))
+ parser.add_argument(
+ '--last_date',
+ help='The earliest date that we care about. All test cases from here '
+ 'on will be picked up. Format is YYYY-MM-DD.')
+ parser.add_argument(
+ '--4c', dest='forcey', required=True, help='Path to a 4c client binary')
+ opts = parser.parse_args(argv)
+
+ forcey = opts.forcey
+ state_file = opts.state_file
+ last_date_str = opts.last_date
+
+ os.makedirs(os.path.dirname(state_file), 0o755, exist_ok=True)
+
+ if last_date_str is None:
+ with open(state_file, encoding='utf-8') as f:
+ data = json.load(f)
+ most_recent_date = from_ymd(data['most_recent_date'])
+ submit_new_test_cases(
+ last_seen_test_cases=data['already_seen'],
+ # Note that we always subtract one day from this to avoid a race:
+ # uploads may appear slightly out-of-order (or builders may lag, or
+ # ...), so the last test-case uploaded for 2020/01/01 might appear
+ # _after_ the first test-case for 2020/01/02. Assuming that builders
+ # won't lag behind for over a day, the easiest way to handle this is to
+ # always check the previous and current days.
+ earliest_date_to_check=most_recent_date - datetime.timedelta(days=1),
+ forcey=forcey,
+ state_file_path=state_file,
+ )
+ else:
+ submit_new_test_cases(
+ last_seen_test_cases=(),
+ earliest_date_to_check=from_ymd(last_date_str),
+ forcey=forcey,
+ state_file_path=state_file,
+ )
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/llvm_tools/upload_lexan_crashes_to_forcey_test.py b/llvm_tools/upload_lexan_crashes_to_forcey_test.py
new file mode 100755
index 00000000..3c9c0d4b
--- /dev/null
+++ b/llvm_tools/upload_lexan_crashes_to_forcey_test.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for upload_lexan_crashes_to_forcey."""
+
+import datetime
+import unittest
+import unittest.mock
+
+import upload_lexan_crashes_to_forcey
+
+
+class Test(unittest.TestCase):
+ """Tests for upload_lexan_crashes_to_forcey."""
+
+ def test_date_parsing_functions(self):
+ self.assertEqual(
+ datetime.date(2020, 2, 1),
+ upload_lexan_crashes_to_forcey.from_ymd('2020-02-01'))
+
+ @unittest.mock.patch(
+ 'upload_lexan_crashes_to_forcey.test_cases_on_or_after',
+ return_value=(
+ (
+ datetime.date(2020, 1, 1),
+ ('gs://test-case-1', 'gs://test-case-1.1'),
+ ),
+ (datetime.date(2020, 1, 2), ('gs://test-case-2',)),
+ (datetime.date(2020, 1, 1), ('gs://test-case-3',)),
+ (datetime.date(2020, 1, 4), ('gs://test-case-4',)),
+ ))
+ @unittest.mock.patch('upload_lexan_crashes_to_forcey.submit_test_case')
+ @unittest.mock.patch('upload_lexan_crashes_to_forcey.persist_state')
+ def test_new_test_case_submission_functions(self, persist_state_mock,
+ submit_test_case_mock,
+ test_cases_on_or_after_mock):
+ forcey_path = '/path/to/4c'
+ real_state_file_path = '/path/to/state/file'
+ earliest_date = datetime.date(2020, 1, 1)
+
+ persist_state_calls = []
+
+ # Since the set this gets is mutated, we need to copy it somehow.
+ def persist_state_side_effect(test_cases_to_persist, state_file_path,
+ most_recent_date):
+ self.assertEqual(state_file_path, real_state_file_path)
+ persist_state_calls.append(
+ (sorted(test_cases_to_persist), most_recent_date))
+
+ persist_state_mock.side_effect = persist_state_side_effect
+
+ upload_lexan_crashes_to_forcey.submit_new_test_cases(
+ last_seen_test_cases=(
+ 'gs://test-case-0',
+ 'gs://test-case-1',
+ ),
+ earliest_date_to_check=earliest_date,
+ forcey=forcey_path,
+ state_file_path=real_state_file_path,
+ )
+
+ test_cases_on_or_after_mock.assert_called_once_with(earliest_date)
+ self.assertEqual(submit_test_case_mock.call_args_list, [
+ unittest.mock.call('gs://test-case-1.1', forcey_path),
+ unittest.mock.call('gs://test-case-2', forcey_path),
+ unittest.mock.call('gs://test-case-3', forcey_path),
+ unittest.mock.call('gs://test-case-4', forcey_path),
+ ])
+
+ self.assertEqual(persist_state_calls, [
+ (
+ ['gs://test-case-0', 'gs://test-case-1', 'gs://test-case-1.1'],
+ datetime.date(2020, 1, 1),
+ ),
+ (
+ [
+ 'gs://test-case-0',
+ 'gs://test-case-1',
+ 'gs://test-case-1.1',
+ 'gs://test-case-2',
+ ],
+ datetime.date(2020, 1, 2),
+ ),
+ (
+ [
+ 'gs://test-case-0',
+ 'gs://test-case-1',
+ 'gs://test-case-1.1',
+ 'gs://test-case-2',
+ 'gs://test-case-3',
+ ],
+ datetime.date(2020, 1, 2),
+ ),
+ (
+ [
+ 'gs://test-case-0',
+ 'gs://test-case-1',
+ 'gs://test-case-1.1',
+ 'gs://test-case-2',
+ 'gs://test-case-3',
+ 'gs://test-case-4',
+ ],
+ datetime.date(2020, 1, 4),
+ ),
+ (
+ [
+ 'gs://test-case-1',
+ 'gs://test-case-1.1',
+ 'gs://test-case-2',
+ 'gs://test-case-3',
+ 'gs://test-case-4',
+ ],
+ datetime.date(2020, 1, 4),
+ ),
+ ])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py
new file mode 100755
index 00000000..cb41eb88
--- /dev/null
+++ b/pgo_tools/monitor_pgo_profiles.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Emails the mage if PGO profile generation hasn't succeeded recently."""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import datetime
+import sys
+import subprocess
+import logging
+from typing import List, NamedTuple, Optional, Tuple
+
+from cros_utils import email_sender
+from cros_utils import tiny_render
+
+PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/'
+ 'pgo-generate-llvm-next-orchestrator')
+
+
+class ProfdataInfo(NamedTuple):
+ """Data about an llvm profdata in our gs:// bucket."""
+ date: datetime.datetime
+ location: str
+
+
+def parse_date(date: str) -> datetime.datetime:
+ time_format = '%Y-%m-%dT%H:%M:%SZ'
+ if not date.endswith('Z'):
+ time_format += '%z'
+ return datetime.datetime.strptime(date, time_format)
+
+
+def fetch_most_recent_profdata(arch: str) -> ProfdataInfo:
+ result = subprocess.run(
+ [
+ 'gsutil.py',
+ 'ls',
+ '-l',
+ f'gs://chromeos-toolchain-artifacts/llvm-pgo/{arch}/'
+ '*.profdata.tar.xz',
+ ],
+ check=True,
+ stdout=subprocess.PIPE,
+ encoding='utf-8',
+ )
+
+ # Each line will be a profdata; the last one is a summary, so drop it.
+ infos = []
+ for rec in result.stdout.strip().splitlines()[:-1]:
+ _size, date, url = rec.strip().split()
+ infos.append(ProfdataInfo(date=parse_date(date), location=url))
+ return max(infos)
+
+
+def compose_complaint_email(
+ out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]]
+) -> Optional[Tuple[str, tiny_render.Piece]]:
+ if not out_of_date_profiles:
+ return None
+
+ if len(out_of_date_profiles) == 1:
+ subject = '1 llvm profile is out of date'
+ body = ['out-of-date profile:']
+ else:
+ subject = f'{len(out_of_date_profiles)} llvm profiles are out of date'
+ body = ['out-of-date profiles:']
+
+ out_of_date_items = []
+ for arch, profdata_info in out_of_date_profiles:
+ out_of_date_items.append(
+ f'{arch} (most recent profile was from {profdata_info.date} at '
+ f'{profdata_info.location!r})')
+
+ body += [
+ tiny_render.UnorderedList(out_of_date_items),
+ tiny_render.line_break,
+ tiny_render.line_break,
+ 'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+ 'Their status can be found at ',
+ tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK),
+ '.',
+ ]
+ return subject, body
+
+
+def main() -> None:
+ logging.basicConfig(level=logging.INFO)
+
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument(
+ '--dry_run',
+ action='store_true',
+ help="Don't actually send an email",
+ )
+ parser.add_argument(
+ '--max_age_days',
+ # These builders run ~weekly. If we fail to generate two in a row,
+ # something's probably wrong.
+ default=15,
+ type=int,
+ help='How old to let profiles get before complaining, in days',
+ )
+ args = parser.parse_args()
+
+ now = datetime.datetime.now()
+ logging.info('Start time is %r', now)
+
+ max_age = datetime.timedelta(days=args.max_age_days)
+ out_of_date_profiles = []
+ for arch in ('arm', 'arm64', 'amd64'):
+ logging.info('Fetching most recent profdata for %r', arch)
+ most_recent = fetch_most_recent_profdata(arch)
+ logging.info('Most recent profdata for %r is %r', arch, most_recent)
+
+ age = now - most_recent.date
+ if age >= max_age:
+ out_of_date_profiles.append((arch, most_recent))
+
+ email = compose_complaint_email(out_of_date_profiles)
+ if not email:
+ logging.info('No email to send; quit')
+ return
+
+ subject, body = email
+
+ identifier = 'llvm-pgo-monitor'
+ subject = f'[{identifier}] {subject}'
+
+ logging.info('Sending email with title %r', subject)
+ if args.dry_run:
+ logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject,
+ tiny_render.render_text_pieces(body))
+ else:
+ email_sender.EmailSender().SendX20Email(
+ subject=subject,
+ identifier=identifier,
+ well_known_recipients=['mage'],
+ direct_recipients=['gbiv@google.com'],
+ text_body=tiny_render.render_text_pieces(body),
+ html_body=tiny_render.render_html_pieces(body),
+ )
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/pgo_tools/monitor_pgo_profiles_unittest.py b/pgo_tools/monitor_pgo_profiles_unittest.py
new file mode 100755
index 00000000..b4e085ec
--- /dev/null
+++ b/pgo_tools/monitor_pgo_profiles_unittest.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for monitor_pgo_profiles."""
+
+import datetime
+import subprocess
+import unittest
+import unittest.mock
+
+import monitor_pgo_profiles
+from cros_utils import tiny_render
+
+
+class Test(unittest.TestCase):
+ """Tests for monitor_pgo_profiles."""
+
+ def test_compose_complaint_email_with_zero_out_of_date(self):
+ self.assertIsNone(monitor_pgo_profiles.compose_complaint_email([]))
+
+ def test_compose_complaint_email_with_one_out_of_date(self):
+ profdata_info = monitor_pgo_profiles.ProfdataInfo(
+ date=datetime.datetime(2020, 1, 2, 3, 4, 5),
+ location='gs://somewhere',
+ )
+ result = monitor_pgo_profiles.compose_complaint_email([
+ ('some_arch', profdata_info),
+ ])
+ self.assertEqual(result, ('1 llvm profile is out of date', [
+ 'out-of-date profile:',
+ tiny_render.UnorderedList([
+ f'some_arch (most recent profile was from {profdata_info.date} at '
+ f'{profdata_info.location!r})'
+ ]),
+ tiny_render.line_break,
+ tiny_render.line_break,
+ 'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+ 'Their status can be found at ',
+ tiny_render.Link(
+ href=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+ inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+ ),
+ '.',
+ ]))
+
+ def test_compose_complaint_email_with_two_out_of_date(self):
+ profdata_info_1 = monitor_pgo_profiles.ProfdataInfo(
+ date=datetime.datetime(2020, 1, 2, 3, 4, 5),
+ location='gs://somewhere',
+ )
+ profdata_info_2 = monitor_pgo_profiles.ProfdataInfo(
+ date=datetime.datetime(2020, 3, 2, 1, 4, 5),
+ location='gs://somewhere-else',
+ )
+ result = monitor_pgo_profiles.compose_complaint_email([
+ ('some_arch', profdata_info_1),
+ ('some_other_arch', profdata_info_2),
+ ])
+ self.assertEqual(result, ('2 llvm profiles are out of date', [
+ 'out-of-date profiles:',
+ tiny_render.UnorderedList([
+ f'some_arch (most recent profile was from {profdata_info_1.date} '
+ f'at {profdata_info_1.location!r})',
+ f'some_other_arch (most recent profile was from '
+ f'{profdata_info_2.date} at {profdata_info_2.location!r})'
+ ]),
+ tiny_render.line_break,
+ tiny_render.line_break,
+ 'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+ 'Their status can be found at ',
+ tiny_render.Link(
+ href=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+ inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+ ),
+ '.',
+ ]))
+
+ @unittest.mock.patch.object(subprocess, 'run')
+ def test_fetching_profdata_functions(self, subprocess_run_mock):
+ ls_return_value = unittest.mock.MagicMock()
+ ls_return_value.stdout = '\n'.join((
+ ' 1234 2020-06-26T05:26:40Z gs://bar',
+ ' 44 2020-06-23T05:26:40Z gs://foo',
+ ' 1234 2020-06-25T05:26:40Z gs://zzz',
+ ))
+ subprocess_run_mock.return_value = ls_return_value
+
+ most_recent = monitor_pgo_profiles.fetch_most_recent_profdata('arm')
+ self.assertEqual(
+ most_recent,
+ monitor_pgo_profiles.ProfdataInfo(
+ date=datetime.datetime(2020, 6, 26, 5, 26, 40),
+ location='gs://bar',
+ ))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
new file mode 100755
index 00000000..3c0ad012
--- /dev/null
+++ b/rust_tools/rust_uprev.py
@@ -0,0 +1,673 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tool to automatically generate a new Rust uprev CL.
+
+This tool is intended to automatically generate a CL to uprev Rust to a
+newer version in Chrome OS, including creating a new Rust version or
+removing an old version. It's based on
+src/third_party/chromiumos-overlay/dev-lang/rust/UPGRADE.md. When using
+the tool, the progress can be saved to a JSON file, so the user can resume
+the process after a failing step is fixed. Example usage to create a new
+version:
+
+1. (inside chroot) $ ./rust_tools/rust_uprev.py
+ --state_file /tmp/state-file.json
+ create --rust_version 1.45.0
+2. Step "compile rust" failed due to the patches can't apply to new version
+3. Manually fix the patches
+4. Execute the command in step 1 again.
+5. Iterate 1-4 for each failed step until the tool passes.
+
+Replace `create --rust_version 1.45.0` with `remove --rust_version 1.43.0`
+if you want to remove all 1.43.0 related stuff in the same CL. Remember to
+use a different state file if you choose to run different subcommands.
+
+If you want a hammer that can do everything for you, use the subcommand
+`roll`. It can create a Rust uprev CL with `create` and `remove` and upload
+the CL to chromium code review.
+
+See `--help` for all available options.
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import pathlib
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple
+
+from llvm_tools import chroot, git
+RUST_PATH = '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust'
+
+
+def get_command_output(command: List[str], *args, **kwargs) -> str:
+ return subprocess.check_output(
+ command, encoding='utf-8', *args, **kwargs).strip()
+
+
+class RustVersion(NamedTuple):
+ """NamedTuple represents a Rust version"""
+ major: int
+ minor: int
+ patch: int
+
+ def __str__(self):
+ return f'{self.major}.{self.minor}.{self.patch}'
+
+ @staticmethod
+ def parse_from_ebuild(ebuild_name: str) -> 'RustVersion':
+ input_re = re.compile(r'^rust-'
+ r'(?P<major>\d+)\.'
+ r'(?P<minor>\d+)\.'
+ r'(?P<patch>\d+)'
+ r'(:?-r\d+)?'
+ r'\.ebuild$')
+ m = input_re.match(ebuild_name)
+ assert m, f'failed to parse {ebuild_name!r}'
+ return RustVersion(
+ int(m.group('major')), int(m.group('minor')), int(m.group('patch')))
+
+ @staticmethod
+ def parse(x: str) -> 'RustVersion':
+ input_re = re.compile(r'^(?:rust-)?'
+ r'(?P<major>\d+)\.'
+ r'(?P<minor>\d+)\.'
+ r'(?P<patch>\d+)'
+ r'(?:.ebuild)?$')
+ m = input_re.match(x)
+ assert m, f'failed to parse {x!r}'
+ return RustVersion(
+ int(m.group('major')), int(m.group('minor')), int(m.group('patch')))
+
+
+def parse_commandline_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument(
+ '--state_file',
+ required=True,
+ help='A state file to hold previous completed steps. If the file '
+ 'exists, it needs to be used together with --continue or --restart. '
+ 'If not exist (do not use --continue in this case), we will create a '
+ 'file for you.',
+ )
+ parser.add_argument(
+ '--restart',
+ action='store_true',
+ help='Restart from the first step. Ignore the completed steps in '
+ 'the state file',
+ )
+ parser.add_argument(
+ '--continue',
+ dest='cont',
+ action='store_true',
+ help='Continue the steps from the state file',
+ )
+
+ create_parser_template = argparse.ArgumentParser(add_help=False)
+ create_parser_template.add_argument(
+ '--template',
+ type=RustVersion.parse,
+ default=None,
+ help='A template to use for creating a Rust uprev from, in the form '
+ 'a.b.c The ebuild has to exist in the chroot. If not specified, the '
+ 'tool will use the current Rust version in the chroot as template.',
+ )
+ create_parser_template.add_argument(
+ '--skip_compile',
+ action='store_true',
+ help='Skip compiling rust to test the tool. Only for testing',
+ )
+
+ subparsers = parser.add_subparsers(dest='subparser_name')
+ subparser_names = []
+ subparser_names.append('create')
+ create_parser = subparsers.add_parser(
+ 'create',
+ parents=[create_parser_template],
+ help='Create changes uprevs Rust to a new version',
+ )
+ create_parser.add_argument(
+ '--rust_version',
+ type=RustVersion.parse,
+ required=True,
+ help='Rust version to uprev to, in the form a.b.c',
+ )
+
+ subparser_names.append('remove')
+ remove_parser = subparsers.add_parser(
+ 'remove',
+ help='Clean up old Rust version from chroot',
+ )
+ remove_parser.add_argument(
+ '--rust_version',
+ type=RustVersion.parse,
+ default=None,
+ help='Rust version to remove, in the form a.b.c If not '
+ 'specified, the tool will remove the oldest version in the chroot',
+ )
+
+ subparser_names.append('roll')
+ roll_parser = subparsers.add_parser(
+ 'roll',
+ parents=[create_parser_template],
+ help='A command can create and upload a Rust uprev CL, including '
+ 'preparing the repo, creating new Rust uprev, deleting old uprev, '
+ 'and upload a CL to crrev.',
+ )
+ roll_parser.add_argument(
+ '--uprev',
+ type=RustVersion.parse,
+ required=True,
+ help='Rust version to uprev to, in the form a.b.c',
+ )
+ roll_parser.add_argument(
+ '--remove',
+ type=RustVersion.parse,
+ default=None,
+ help='Rust version to remove, in the form a.b.c If not '
+ 'specified, the tool will remove the oldest version in the chroot',
+ )
+ roll_parser.add_argument(
+ '--skip_cross_compiler',
+ action='store_true',
+ help='Skip updating cross-compiler in the chroot',
+ )
+ roll_parser.add_argument(
+ '--no_upload',
+ action='store_true',
+ help='If specified, the tool will not upload the CL for review',
+ )
+
+ args = parser.parse_args()
+ if args.subparser_name not in subparser_names:
+ parser.error('one of %s must be specified' % subparser_names)
+
+ if args.cont and args.restart:
+ parser.error('Please select either --continue or --restart')
+
+ if os.path.exists(args.state_file):
+ if not args.cont and not args.restart:
+ parser.error('State file exists, so you should either --continue '
+ 'or --restart')
+ if args.cont and not os.path.exists(args.state_file):
+ parser.error('Indicate --continue but the state file does not exist')
+
+ if args.restart and os.path.exists(args.state_file):
+ os.remove(args.state_file)
+
+ return args
+
+
+def parse_stage0_file(new_version: RustVersion) -> Tuple[str, str, str]:
+ # Find stage0 date, rustc and cargo
+ stage0_file = get_command_output([
+ 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/'
+ f'{new_version}/src/stage0.txt'
+ ])
+ regexp = re.compile(r'date:\s*(?P<date>\d+-\d+-\d+)\s+'
+ r'rustc:\s*(?P<rustc>\d+\.\d+\.\d+)\s+'
+ r'cargo:\s*(?P<cargo>\d+\.\d+\.\d+)')
+ m = regexp.search(stage0_file)
+ assert m, 'failed to parse stage0.txt file'
+ stage0_date, stage0_rustc, stage0_cargo = m.groups()
+ logging.info('Found stage0 file has date: %s, rustc: %s, cargo: %s',
+ stage0_date, stage0_rustc, stage0_cargo)
+ return stage0_date, stage0_rustc, stage0_cargo
+
+
+def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion]
+ ) -> Optional[Tuple[RustVersion, str]]:
+ if template is None:
+ ebuild_path = get_command_output(['equery', 'w', 'rust'])
+ ebuild_name = os.path.basename(ebuild_path)
+ template_version = RustVersion.parse_from_ebuild(ebuild_name)
+ else:
+ ebuild_path = find_ebuild_for_rust_version(template)
+ template_version = template
+
+ if rust_version <= template_version:
+ logging.info(
+ 'Requested version %s is not newer than the template version %s.',
+ rust_version, template_version)
+ return None
+
+ logging.info('Template Rust version is %s (ebuild: %r)', template_version,
+ ebuild_path)
+ return template_version, ebuild_path
+
+
+def copy_patches(template_version: RustVersion,
+ new_version: RustVersion) -> None:
+ patch_path = os.path.join(RUST_PATH, 'files')
+ for f in os.listdir(patch_path):
+ if f'rust-{template_version}' not in f:
+ continue
+ logging.info('Rename patch %s to new version', f)
+ new_name = f.replace(str(template_version), str(new_version))
+ shutil.copyfile(
+ os.path.join(patch_path, f),
+ os.path.join(patch_path, new_name),
+ )
+
+ subprocess.check_call(['git', 'add', f'files/rust-{new_version}-*.patch'],
+ cwd=RUST_PATH)
+
+
+def create_ebuild(template_ebuild: str, new_version: RustVersion) -> str:
+ shutil.copyfile(template_ebuild,
+ os.path.join(RUST_PATH, f'rust-{new_version}.ebuild'))
+ subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'],
+ cwd=RUST_PATH)
+ return os.path.join(RUST_PATH, f'rust-{new_version}.ebuild')
+
+
+def update_ebuild(ebuild_file: str, stage0_info: Tuple[str, str, str]) -> None:
+ stage0_date, stage0_rustc, stage0_cargo = stage0_info
+ with open(ebuild_file, encoding='utf-8') as f:
+ contents = f.read()
+ # Update STAGE0_DATE in the ebuild
+ stage0_date_re = re.compile(r'STAGE0_DATE="(\d+-\d+-\d+)"')
+ if not stage0_date_re.search(contents):
+ raise RuntimeError('STAGE0_DATE not found in rust ebuild')
+ new_contents = stage0_date_re.sub(f'STAGE0_DATE="{stage0_date}"', contents)
+
+ # Update STAGE0_VERSION in the ebuild
+ stage0_rustc_re = re.compile(r'STAGE0_VERSION="[^"]*"')
+ if not stage0_rustc_re.search(new_contents):
+ raise RuntimeError('STAGE0_VERSION not found in rust ebuild')
+ new_contents = stage0_rustc_re.sub(f'STAGE0_VERSION="{stage0_rustc}"',
+ new_contents)
+
+ # Update STAGE0_VERSION_CARGO in the ebuild
+ stage0_cargo_re = re.compile(r'STAGE0_VERSION_CARGO="[^"]*"')
+ if not stage0_cargo_re.search(new_contents):
+ raise RuntimeError('STAGE0_VERSION_CARGO not found in rust ebuild')
+ new_contents = stage0_cargo_re.sub(f'STAGE0_VERSION_CARGO="{stage0_cargo}"',
+ new_contents)
+ with open(ebuild_file, 'w', encoding='utf-8') as f:
+ f.write(new_contents)
+ logging.info(
+ 'Rust ebuild file has STAGE0_DATE, STAGE0_VERSION, STAGE0_VERSION_CARGO '
+ 'updated to %s, %s, %s respectively', stage0_date, stage0_rustc,
+ stage0_cargo)
+
+
+def flip_mirror_in_ebuild(ebuild_file: str, add: bool) -> None:
+ restrict_re = re.compile(
+ r'(?P<before>RESTRICT=")(?P<values>"[^"]*"|.*)(?P<after>")')
+ with open(ebuild_file, encoding='utf-8') as f:
+ contents = f.read()
+ m = restrict_re.search(contents)
+ assert m, 'failed to find RESTRICT variable in Rust ebuild'
+ values = m.group('values')
+ if add:
+ if 'mirror' in values:
+ return
+ values += ' mirror'
+ else:
+ if 'mirror' not in values:
+ return
+ values = values.replace(' mirror', '')
+ new_contents = restrict_re.sub(r'\g<before>%s\g<after>' % values, contents)
+ with open(ebuild_file, 'w', encoding='utf-8') as f:
+ f.write(new_contents)
+
+
+def rust_ebuild_actions(actions: List[str], sudo: bool = False) -> None:
+ ebuild_path_inchroot = get_command_output(['equery', 'w', 'rust'])
+ cmd = ['ebuild', ebuild_path_inchroot] + actions
+ if sudo:
+ cmd = ['sudo'] + cmd
+ subprocess.check_call(cmd)
+
+
+def update_manifest(ebuild_file: str) -> None:
+ logging.info('Added "mirror" to RESTRICT to Rust ebuild')
+ flip_mirror_in_ebuild(ebuild_file, add=True)
+ rust_ebuild_actions(['manifest'])
+ logging.info('Removed "mirror" to RESTRICT from Rust ebuild')
+ flip_mirror_in_ebuild(ebuild_file, add=False)
+
+
+def update_rust_packages(rust_version: RustVersion, add: bool) -> None:
+ package_file = os.path.join(
+ RUST_PATH, '../../profiles/targets/chromeos/package.provided')
+ with open(package_file, encoding='utf-8') as f:
+ contents = f.read()
+ if add:
+ rust_packages_re = re.compile(r'dev-lang/rust-(\d+\.\d+\.\d+)')
+ rust_packages = rust_packages_re.findall(contents)
+ # Assume all the rust packages are in alphabetical order, so insert the new
+ # version to the place after the last rust_packages
+ new_str = f'dev-lang/rust-{rust_version}'
+ new_contents = contents.replace(rust_packages[-1],
+ f'{rust_packages[-1]}\n{new_str}')
+ logging.info('%s has been inserted into package.provided', new_str)
+ else:
+ old_str = f'dev-lang/rust-{rust_version}\n'
+ assert old_str in contents, f'{old_str!r} not found in package.provided'
+ new_contents = contents.replace(old_str, '')
+ logging.info('%s has been removed from package.provided', old_str)
+
+ with open(package_file, 'w', encoding='utf-8') as f:
+ f.write(new_contents)
+
+
+def update_virtual_rust(template_version: RustVersion,
+ new_version: RustVersion) -> None:
+ virtual_rust_dir = os.path.join(RUST_PATH, '../../virtual/rust')
+ assert os.path.exists(virtual_rust_dir)
+ shutil.copyfile(
+ os.path.join(virtual_rust_dir, f'rust-{template_version}.ebuild'),
+ os.path.join(virtual_rust_dir, f'rust-{new_version}.ebuild'))
+ subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'],
+ cwd=virtual_rust_dir)
+
+
+def upload_single_tarball(rust_url: str, tarfile_name: str,
+ tempdir: str) -> None:
+ rust_src = f'{rust_url}/{tarfile_name}'
+ gsutil_location = f'gs://chromeos-localmirror/distfiles/{tarfile_name}'
+
+ missing_file = subprocess.call(
+ ['gsutil', 'ls', gsutil_location],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ if not missing_file:
+ logging.info('Rust artifact at %s already exists; skipping download',
+ gsutil_location)
+ return
+
+ logging.info('Downloading Rust artifact from %s', rust_src)
+
+ # Download Rust's source
+ rust_file = os.path.join(tempdir, tarfile_name)
+ subprocess.check_call(['curl', '-f', '-o', rust_file, rust_src])
+
+ # Verify the signature of the source
+ sig_file = os.path.join(tempdir, 'rustc_sig.asc')
+ subprocess.check_call(['curl', '-f', '-o', sig_file, f'{rust_src}.asc'])
+ try:
+ subprocess.check_output(['gpg', '--verify', sig_file, rust_file],
+ encoding='utf-8',
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ if "gpg: Can't check signature" not in e.output:
+ raise RuntimeError(f'Failed to execute `gpg --verify`, {e.output}')
+
+ # If it fails to verify the signature, try import rustc key, and retry.
+ keys = get_command_output(
+ ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc'])
+ subprocess.run(['gpg', '--import'],
+ input=keys,
+ encoding='utf-8',
+ check=True)
+ subprocess.check_call(['gpg', '--verify', sig_file, rust_file])
+
+ # Since we are using `-n` to skip an item if it already exists, there's no
+ # need to check if the file exists on GS bucket or not.
+ subprocess.check_call(
+ ['gsutil', 'cp', '-n', '-a', 'public-read', rust_file, gsutil_location])
+
+
+def upload_to_localmirror(tempdir: str, rust_version: RustVersion,
+ stage0_info: Tuple[str, str, str]) -> None:
+ stage0_date, stage0_rustc, stage0_cargo = stage0_info
+ rust_url = 'https://static.rust-lang.org/dist'
+ # Upload rustc source
+ upload_single_tarball(
+ rust_url,
+ f'rustc-{rust_version}-src.tar.gz',
+ tempdir,
+ )
+ # Upload stage0 toolchain
+ upload_single_tarball(
+ f'{rust_url}/{stage0_date}',
+ f'rust-std-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz',
+ tempdir,
+ )
+ # Upload stage0 source
+ upload_single_tarball(
+ rust_url,
+ f'rustc-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz',
+ tempdir,
+ )
+ # Upload stage0 cargo
+ upload_single_tarball(
+ rust_url,
+ f'cargo-{stage0_cargo}-x86_64-unknown-linux-gnu.tar.gz',
+ tempdir,
+ )
+
+
+def perform_step(state_file: pathlib.Path,
+ tmp_state_file: pathlib.Path,
+ completed_steps: Dict[str, Any],
+ step_name: str,
+ step_fn: Callable[[], T],
+ result_from_json: Optional[Callable[[Any], T]] = None,
+ result_to_json: Optional[Callable[[T], Any]] = None) -> T:
+ if step_name in completed_steps:
+ logging.info('Skipping previously completed step %s', step_name)
+ if result_from_json:
+ return result_from_json(completed_steps[step_name])
+ return completed_steps[step_name]
+
+ logging.info('Running step %s', step_name)
+ val = step_fn()
+ logging.info('Step %s complete', step_name)
+ if result_to_json:
+ completed_steps[step_name] = result_to_json(val)
+ else:
+ completed_steps[step_name] = val
+
+ with tmp_state_file.open('w', encoding='utf-8') as f:
+ json.dump(completed_steps, f, indent=4)
+ tmp_state_file.rename(state_file)
+ return val
+
+
+def prepare_uprev_from_json(obj: Any) -> Optional[Tuple[RustVersion, str]]:
+ if not obj:
+ return None
+ version, ebuild_path = obj
+ return RustVersion(*version), ebuild_path
+
+
+def create_rust_uprev(rust_version: RustVersion,
+ maybe_template_version: Optional[RustVersion],
+ skip_compile: bool, run_step: Callable[[], T]) -> None:
+ stage0_info = run_step(
+ 'parse stage0 file', lambda: parse_stage0_file(rust_version))
+ template_version, template_ebuild = run_step(
+ 'prepare uprev',
+ lambda: prepare_uprev(rust_version, maybe_template_version),
+ result_from_json=prepare_uprev_from_json,
+ )
+ if template_ebuild is None:
+ return
+
+ run_step('copy patches', lambda: copy_patches(template_version, rust_version))
+ ebuild_file = run_step(
+ 'create ebuild', lambda: create_ebuild(template_ebuild, rust_version))
+ run_step('update ebuild', lambda: update_ebuild(ebuild_file, stage0_info))
+ with tempfile.TemporaryDirectory(dir='/tmp') as tempdir:
+ run_step('upload_to_localmirror', lambda: upload_to_localmirror(
+ tempdir, rust_version, stage0_info))
+ run_step('update manifest to add new version', lambda: update_manifest(
+ ebuild_file))
+ if not skip_compile:
+ run_step('emerge rust', lambda: subprocess.check_call(
+ ['sudo', 'emerge', 'dev-lang/rust']))
+ run_step('insert version into rust packages', lambda: update_rust_packages(
+ rust_version, add=True))
+ run_step('upgrade virtual/rust', lambda: update_virtual_rust(
+ template_version, rust_version))
+
+
+def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]:
+ return [(RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x))
+ for x in os.listdir(RUST_PATH)
+ if x.endswith('.ebuild')]
+
+
+def find_oldest_rust_version_in_chroot() -> Tuple[RustVersion, str]:
+ rust_versions = find_rust_versions_in_chroot()
+ if len(rust_versions) <= 1:
+ raise RuntimeError('Expect to find more than one Rust versions')
+ return min(rust_versions)
+
+
+def find_ebuild_for_rust_version(version: RustVersion) -> str:
+ rust_ebuilds = [
+ ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version
+ ]
+ if not rust_ebuilds:
+ raise ValueError(f'No Rust ebuilds found matching {version}')
+ if len(rust_ebuilds) > 1:
+ raise ValueError(f'Multiple Rust ebuilds found matching {version}: '
+ f'{rust_ebuilds}')
+ return rust_ebuilds[0]
+
+
+def remove_files(filename: str, path: str) -> None:
+ subprocess.check_call(['git', 'rm', filename], cwd=path)
+
+
+def remove_rust_uprev(rust_version: Optional[RustVersion],
+ run_step: Callable[[], T]) -> None:
+
+ def find_desired_rust_version():
+ if rust_version:
+ return rust_version, find_ebuild_for_rust_version(rust_version)
+ return find_oldest_rust_version_in_chroot()
+
+ delete_version, delete_ebuild = run_step(
+ 'find rust version to delete',
+ find_desired_rust_version,
+ result_from_json=prepare_uprev_from_json,
+ )
+ run_step(
+ 'remove patches', lambda: remove_files(
+ f'files/rust-{delete_version}-*.patch', RUST_PATH))
+ run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH))
+ ebuild_file = get_command_output(['equery', 'w', 'rust'])
+ run_step('update manifest to delete old version', lambda: update_manifest(
+ ebuild_file))
+ run_step('remove version from rust packages', lambda: update_rust_packages(
+ delete_version, add=False))
+ run_step(
+ 'remove virtual/rust', lambda: remove_files(
+ f'rust-{delete_version}.ebuild',
+ os.path.join(RUST_PATH, '../../virtual/rust')))
+
+
+def create_new_repo(rust_version: RustVersion) -> None:
+ output = get_command_output(['git', 'status', '--porcelain'], cwd=RUST_PATH)
+ if output:
+ raise RuntimeError(
+ f'{RUST_PATH} has uncommitted changes, please either discard them '
+ 'or commit them.')
+ git.CreateBranch(RUST_PATH, f'rust-to-{rust_version}')
+
+
+def build_cross_compiler() -> None:
+ # Get target triples in ebuild
+ rust_ebuild = get_command_output(['equery', 'w', 'rust'])
+ with open(rust_ebuild, encoding='utf-8') as f:
+ contents = f.read()
+
+ target_triples_re = re.compile(r'RUSTC_TARGET_TRIPLES=\(([^)]+)\)')
+ m = target_triples_re.search(contents)
+ assert m, 'RUST_TARGET_TRIPLES not found in rust ebuild'
+ target_triples = m.group(1).strip().split('\n')
+
+ compiler_targets_to_install = [
+ target.strip() for target in target_triples if 'cros-' in target
+ ]
+ for target in target_triples:
+ if 'cros-' not in target:
+ continue
+ target = target.strip()
+
+ # We also always need arm-none-eabi, though it's not mentioned in
+ # RUSTC_TARGET_TRIPLES.
+ compiler_targets_to_install.append('arm-none-eabi')
+
+ logging.info('Emerging cross compilers %s', compiler_targets_to_install)
+ subprocess.check_call(
+ ['sudo', 'emerge', '-j', '-G'] +
+ [f'cross-{target}/gcc' for target in compiler_targets_to_install])
+
+
+def create_new_commit(rust_version: RustVersion) -> None:
+ subprocess.check_call(['git', 'add', '-A'], cwd=RUST_PATH)
+ messages = [
+ f'[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}',
+ '',
+ 'This CL is created by rust_uprev tool automatically.'
+ '',
+ 'BUG=None',
+ 'TEST=Use CQ to test the new Rust version',
+ ]
+ git.UploadChanges(RUST_PATH, f'rust-to-{rust_version}', messages)
+
+
+def main() -> None:
+ if not chroot.InChroot():
+ raise RuntimeError('This script must be executed inside chroot')
+
+ logging.basicConfig(level=logging.INFO)
+
+ args = parse_commandline_args()
+
+ state_file = pathlib.Path(args.state_file)
+ tmp_state_file = state_file.with_suffix('.tmp')
+
+ try:
+ with state_file.open(encoding='utf-8') as f:
+ completed_steps = json.load(f)
+ except FileNotFoundError:
+ completed_steps = {}
+
+ def run_step(
+ step_name: str,
+ step_fn: Callable[[], T],
+ result_from_json: Optional[Callable[[Any], T]] = None,
+ result_to_json: Optional[Callable[[T], Any]] = None,
+ ) -> T:
+ return perform_step(state_file, tmp_state_file, completed_steps, step_name,
+ step_fn, result_from_json, result_to_json)
+
+ if args.subparser_name == 'create':
+ create_rust_uprev(args.rust_version, args.template, args.skip_compile,
+ run_step)
+ elif args.subparser_name == 'remove':
+ remove_rust_uprev(args.rust_version, run_step)
+ else:
+ # If you have added more subparser_name, please also add the handlers above
+ assert args.subparser_name == 'roll'
+ run_step('create new repo', lambda: create_new_repo(args.uprev))
+ if not args.skip_cross_compiler:
+ run_step('build cross compiler', build_cross_compiler)
+ create_rust_uprev(args.uprev, args.template, args.skip_compile, run_step)
+ remove_rust_uprev(args.remove, run_step)
+ if not args.no_upload:
+ run_step('create rust uprev CL', lambda: create_new_commit(args.uprev))
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
new file mode 100755
index 00000000..fc506004
--- /dev/null
+++ b/rust_tools/rust_uprev_test.py
@@ -0,0 +1,455 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for rust_uprev.py"""
+
+# pylint: disable=cros-logging-import
+import os
+import shutil
+import subprocess
+import unittest
+from unittest import mock
+
+from llvm_tools import git
+
+import rust_uprev
+
+
+class RustVersionTest(unittest.TestCase):
+ """Tests for RustVersion class"""
+
+ def test_str(self):
+ obj = rust_uprev.RustVersion(major=1, minor=2, patch=3)
+ self.assertEqual(str(obj), '1.2.3')
+
+ def test_parse_version_only(self):
+ expected = rust_uprev.RustVersion(major=1, minor=2, patch=3)
+ actual = rust_uprev.RustVersion.parse('1.2.3')
+ self.assertEqual(expected, actual)
+
+ def test_parse_ebuild_name(self):
+ expected = rust_uprev.RustVersion(major=1, minor=2, patch=3)
+ actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3.ebuild')
+ self.assertEqual(expected, actual)
+
+ actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3-r1.ebuild')
+ self.assertEqual(expected, actual)
+
+ def test_parse_fail(self):
+ with self.assertRaises(AssertionError) as context:
+ rust_uprev.RustVersion.parse('invalid-rust-1.2.3')
+ self.assertEqual("failed to parse 'invalid-rust-1.2.3'",
+ str(context.exception))
+
+
+class PrepareUprevTest(unittest.TestCase):
+ """Tests for prepare_uprev step in rust_uprev"""
+
+ def setUp(self):
+ self.version_old = rust_uprev.RustVersion(1, 2, 3)
+ self.version_new = rust_uprev.RustVersion(1, 3, 5)
+
+ @mock.patch.object(
+ rust_uprev,
+ 'find_ebuild_for_rust_version',
+ return_value='/path/to/ebuild')
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ def test_success_with_template(self, mock_command, mock_find_ebuild):
+ expected = (self.version_old, '/path/to/ebuild')
+ actual = rust_uprev.prepare_uprev(
+ rust_version=self.version_new, template=self.version_old)
+ self.assertEqual(expected, actual)
+ mock_command.assert_not_called()
+
+ @mock.patch.object(
+ rust_uprev,
+ 'find_ebuild_for_rust_version',
+ return_value='/path/to/ebuild')
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ def test_return_none_with_template_larger_than_input(self, mock_command,
+ _mock_find_ebuild):
+ ret = rust_uprev.prepare_uprev(
+ rust_version=self.version_old, template=self.version_new)
+ self.assertIsNone(ret)
+ mock_command.assert_not_called()
+
+ @mock.patch.object(os.path, 'exists')
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ def test_success_without_template(self, mock_command, mock_exists):
+ rust_ebuild_path = f'/path/to/rust/rust-{self.version_old}-r3.ebuild'
+ mock_command.return_value = rust_ebuild_path
+ expected = (self.version_old, rust_ebuild_path)
+ actual = rust_uprev.prepare_uprev(
+ rust_version=self.version_new, template=None)
+ self.assertEqual(expected, actual)
+ mock_command.assert_called_once_with(['equery', 'w', 'rust'])
+ mock_exists.assert_not_called()
+
+ @mock.patch.object(os.path, 'exists')
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ def test_return_none_with_ebuild_larger_than_input(self, mock_command,
+ mock_exists):
+ mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild'
+ ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=None)
+ self.assertIsNone(ret)
+ mock_exists.assert_not_called()
+
+ def test_prepare_uprev_from_json(self):
+ ebuild_path = '/path/to/the/ebuild'
+ json_result = (list(self.version_new), ebuild_path)
+ expected = (self.version_new, ebuild_path)
+ actual = rust_uprev.prepare_uprev_from_json(json_result)
+ self.assertEqual(expected, actual)
+
+
+class UpdateEbuildTest(unittest.TestCase):
+ """Tests for update_ebuild step in rust_uprev"""
+ ebuild_file_before = """
+ STAGE0_DATE="2019-01-01"
+ STAGE0_VERSION="any.random.(number)"
+ STAGE0_VERSION_CARGO="0.0.0"
+ """
+ ebuild_file_after = """
+ STAGE0_DATE="2020-01-01"
+ STAGE0_VERSION="1.1.1"
+ STAGE0_VERSION_CARGO="0.1.0"
+ """
+
+ def test_success(self):
+ mock_open = mock.mock_open(read_data=self.ebuild_file_before)
+ ebuild_file = '/path/to/rust/rust-1.3.5.ebuild'
+ with mock.patch('builtins.open', mock_open):
+ rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0'))
+ mock_open.return_value.__enter__().write.assert_called_once_with(
+ self.ebuild_file_after)
+
+ def test_fail_when_ebuild_misses_a_variable(self):
+ ebuild_file = 'STAGE0_DATE="2019-01-01"'
+ mock_open = mock.mock_open(read_data=ebuild_file)
+ ebuild_file = '/path/to/rust/rust-1.3.5.ebuild'
+ with mock.patch('builtins.open', mock_open):
+ with self.assertRaises(RuntimeError) as context:
+ rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0'))
+ self.assertEqual('STAGE0_VERSION not found in rust ebuild',
+ str(context.exception))
+
+
+class UpdateManifestTest(unittest.TestCase):
+ """Tests for update_manifest step in rust_uprev"""
+
+ # pylint: disable=protected-access
+ def _run_test_flip_mirror(self, before, after, add, expect_write):
+ mock_open = mock.mock_open(read_data=f'RESTRICT="{before}"')
+ with mock.patch('builtins.open', mock_open):
+ rust_uprev.flip_mirror_in_ebuild('', add=add)
+ if expect_write:
+ mock_open.return_value.__enter__().write.assert_called_once_with(
+ f'RESTRICT="{after}"')
+
+ def test_add_mirror_in_ebuild(self):
+ self._run_test_flip_mirror(
+ before='variable1 variable2',
+ after='variable1 variable2 mirror',
+ add=True,
+ expect_write=True)
+
+ def test_remove_mirror_in_ebuild(self):
+ self._run_test_flip_mirror(
+ before='variable1 variable2 mirror',
+ after='variable1 variable2',
+ add=False,
+ expect_write=True)
+
+ def test_add_mirror_when_exists(self):
+ self._run_test_flip_mirror(
+ before='variable1 variable2 mirror',
+ after='variable1 variable2 mirror',
+ add=True,
+ expect_write=False)
+
+ def test_remove_mirror_when_not_exists(self):
+ self._run_test_flip_mirror(
+ before='variable1 variable2',
+ after='variable1 variable2',
+ add=False,
+ expect_write=False)
+
+ @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild')
+ @mock.patch.object(rust_uprev, 'rust_ebuild_actions')
+ def test_update_manifest(self, mock_run, mock_flip):
+ ebuild_file = '/path/to/rust/rust-1.1.1.ebuild'
+ rust_uprev.update_manifest(ebuild_file)
+ mock_run.assert_called_once_with(['manifest'])
+ mock_flip.assert_has_calls(
+ [mock.call(ebuild_file, add=True),
+ mock.call(ebuild_file, add=False)])
+
+
+class UpdateRustPackagesTests(unittest.TestCase):
+ """Tests for update_rust_packages step."""
+
+ def setUp(self):
+ self.old_version = rust_uprev.RustVersion(1, 1, 0)
+ self.current_version = rust_uprev.RustVersion(1, 2, 3)
+ self.new_version = rust_uprev.RustVersion(1, 3, 5)
+ self.ebuild_file = os.path.join(rust_uprev.RUST_PATH,
+ 'rust-{self.new_version}.ebuild')
+
+ def test_add_new_rust_packages(self):
+ package_before = (f'dev-lang/rust-{self.old_version}\n'
+ f'dev-lang/rust-{self.current_version}')
+ package_after = (f'dev-lang/rust-{self.old_version}\n'
+ f'dev-lang/rust-{self.current_version}\n'
+ f'dev-lang/rust-{self.new_version}')
+ mock_open = mock.mock_open(read_data=package_before)
+ with mock.patch('builtins.open', mock_open):
+ rust_uprev.update_rust_packages(self.new_version, add=True)
+ mock_open.return_value.__enter__().write.assert_called_once_with(
+ package_after)
+
+ def test_remove_old_rust_packages(self):
+ package_before = (f'dev-lang/rust-{self.old_version}\n'
+ f'dev-lang/rust-{self.current_version}\n'
+ f'dev-lang/rust-{self.new_version}')
+ package_after = (f'dev-lang/rust-{self.current_version}\n'
+ f'dev-lang/rust-{self.new_version}')
+ mock_open = mock.mock_open(read_data=package_before)
+ with mock.patch('builtins.open', mock_open):
+ rust_uprev.update_rust_packages(self.old_version, add=False)
+ mock_open.return_value.__enter__().write.assert_called_once_with(
+ package_after)
+
+
+class UploadToLocalmirrorTests(unittest.TestCase):
+ """Tests for upload_to_localmirror"""
+
+ def setUp(self):
+ self.tempdir = '/tmp/any/dir'
+ self.new_version = rust_uprev.RustVersion(1, 3, 5)
+ self.rust_url = 'https://static.rust-lang.org/dist'
+ self.tarfile_name = f'rustc-{self.new_version}-src.tar.gz'
+ self.rust_src = f'https://static.rust-lang.org/dist/{self.tarfile_name}'
+ self.gsurl = f'gs://chromeos-localmirror/distfiles/{self.tarfile_name}'
+ self.rust_file = os.path.join(self.tempdir, self.tarfile_name)
+ self.sig_file = os.path.join(self.tempdir, 'rustc_sig.asc')
+
+ @mock.patch.object(subprocess, 'call', return_value=1)
+ @mock.patch.object(subprocess, 'check_call')
+ @mock.patch.object(subprocess, 'check_output')
+ @mock.patch.object(subprocess, 'run')
+ def test_pass_without_retry(self, mock_run, mock_output, mock_call,
+ mock_raw_call):
+ rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name,
+ self.tempdir)
+ mock_output.assert_called_once_with(
+ ['gpg', '--verify', self.sig_file, self.rust_file],
+ encoding='utf-8',
+ stderr=subprocess.STDOUT)
+ mock_raw_call.assert_has_calls([
+ mock.call(['gsutil', 'ls', self.gsurl],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL)
+ ])
+ mock_call.assert_has_calls([
+ mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]),
+ mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']),
+ mock.call([
+ 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file,
+ self.gsurl
+ ])
+ ])
+ mock_run.assert_not_called()
+
+ @mock.patch.object(subprocess, 'call')
+ @mock.patch.object(subprocess, 'check_call')
+ @mock.patch.object(subprocess, 'check_output')
+ @mock.patch.object(subprocess, 'run')
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call,
+ mock_raw_call):
+ mock_check.side_effect = subprocess.CalledProcessError(
+ returncode=2, cmd=None, output="gpg: Can't check signature")
+ mock_output.return_value = 'some_gpg_keys'
+ rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name,
+ self.tempdir)
+ mock_check.assert_called_once_with(
+ ['gpg', '--verify', self.sig_file, self.rust_file],
+ encoding='utf-8',
+ stderr=subprocess.STDOUT)
+ mock_output.assert_called_once_with(
+ ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc'])
+ mock_run.assert_called_once_with(['gpg', '--import'],
+ input='some_gpg_keys',
+ encoding='utf-8',
+ check=True)
+ mock_raw_call.assert_has_calls([
+ mock.call(['gsutil', 'ls', self.gsurl],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL)
+ ])
+ mock_call.assert_has_calls([
+ mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]),
+ mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']),
+ mock.call(['gpg', '--verify', self.sig_file, self.rust_file]),
+ mock.call([
+ 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file,
+ self.gsurl
+ ])
+ ])
+
+ @mock.patch.object(rust_uprev, 'upload_single_tarball')
+ def test_upload_to_mirror(self, mock_upload):
+ stage0_info = '2020-01-01', '1.1.1', '0.1.0'
+ rust_uprev.upload_to_localmirror(self.tempdir, self.new_version,
+ stage0_info)
+ mock_upload.assert_has_calls([
+ mock.call(self.rust_url, f'rustc-{self.new_version}-src.tar.gz',
+ self.tempdir),
+ mock.call(f'{self.rust_url}/{stage0_info[0]}',
+ f'rust-std-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz',
+ self.tempdir),
+ mock.call(self.rust_url,
+ f'rustc-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz',
+ self.tempdir),
+ mock.call(self.rust_url,
+ f'cargo-{stage0_info[2]}-x86_64-unknown-linux-gnu.tar.gz',
+ self.tempdir),
+ ])
+
+
+class RustUprevOtherStagesTests(unittest.TestCase):
+ """Tests for other steps in rust_uprev"""
+
+ def setUp(self):
+ self.old_version = rust_uprev.RustVersion(1, 1, 0)
+ self.current_version = rust_uprev.RustVersion(1, 2, 3)
+ self.new_version = rust_uprev.RustVersion(1, 3, 5)
+ self.ebuild_file = os.path.join(rust_uprev.RUST_PATH,
+ 'rust-{self.new_version}.ebuild')
+
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ def test_parse_stage0_file(self, mock_get):
+ stage0_file = """
+ unrelated stuff before
+ date: 2020-01-01
+ rustc: 1.1.1
+ cargo: 0.1.0
+ unrelated stuff after
+ """
+ mock_get.return_value = stage0_file
+ expected = '2020-01-01', '1.1.1', '0.1.0'
+ rust_version = rust_uprev.RustVersion(1, 2, 3)
+ actual = rust_uprev.parse_stage0_file(rust_version)
+ self.assertEqual(expected, actual)
+ mock_get.assert_called_once_with([
+ 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/'
+ f'{rust_version}/src/stage0.txt'
+ ])
+
+ @mock.patch.object(shutil, 'copyfile')
+ @mock.patch.object(os, 'listdir')
+ @mock.patch.object(subprocess, 'check_call')
+ def test_copy_patches(self, mock_call, mock_ls, mock_copy):
+ mock_ls.return_value = [
+ f'rust-{self.old_version}-patch-1.patch',
+ f'rust-{self.old_version}-patch-2-old.patch',
+ f'rust-{self.current_version}-patch-1.patch',
+ f'rust-{self.current_version}-patch-2-new.patch'
+ ]
+ rust_uprev.copy_patches(self.current_version, self.new_version)
+ mock_copy.assert_has_calls([
+ mock.call(
+ os.path.join(rust_uprev.RUST_PATH, 'files',
+ f'rust-{self.current_version}-patch-1.patch'),
+ os.path.join(rust_uprev.RUST_PATH, 'files',
+ f'rust-{self.new_version}-patch-1.patch'),
+ ),
+ mock.call(
+ os.path.join(rust_uprev.RUST_PATH, 'files',
+ f'rust-{self.current_version}-patch-2-new.patch'),
+ os.path.join(rust_uprev.RUST_PATH, 'files',
+ f'rust-{self.new_version}-patch-2-new.patch'))
+ ])
+ mock_call.assert_called_once_with(
+ ['git', 'add', f'files/rust-{self.new_version}-*.patch'],
+ cwd=rust_uprev.RUST_PATH)
+
+ @mock.patch.object(shutil, 'copyfile')
+ @mock.patch.object(subprocess, 'check_call')
+ def test_create_ebuild(self, mock_call, mock_copy):
+ template_ebuild = f'/path/to/rust-{self.current_version}-r2.ebuild'
+ rust_uprev.create_ebuild(template_ebuild, self.new_version)
+ mock_copy.assert_called_once_with(
+ template_ebuild,
+ os.path.join(rust_uprev.RUST_PATH, f'rust-{self.new_version}.ebuild'))
+ mock_call.assert_called_once_with(
+ ['git', 'add', f'rust-{self.new_version}.ebuild'],
+ cwd=rust_uprev.RUST_PATH)
+
+ @mock.patch.object(os.path, 'exists', return_value=True)
+ @mock.patch.object(shutil, 'copyfile')
+ @mock.patch.object(subprocess, 'check_call')
+ def test_update_virtual_rust(self, mock_call, mock_copy, mock_exists):
+ virtual_rust_dir = os.path.join(rust_uprev.RUST_PATH, '../../virtual/rust')
+ rust_uprev.update_virtual_rust(self.current_version, self.new_version)
+ mock_call.assert_called_once_with(
+ ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=virtual_rust_dir)
+ mock_copy.assert_called_once_with(
+ os.path.join(virtual_rust_dir, f'rust-{self.current_version}.ebuild'),
+ os.path.join(virtual_rust_dir, f'rust-{self.new_version}.ebuild'))
+ mock_exists.assert_called_once_with(virtual_rust_dir)
+
+ @mock.patch.object(os, 'listdir')
+ def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls):
+ oldest_version_name = f'rust-{self.old_version}.ebuild'
+ mock_ls.return_value = [
+ oldest_version_name, f'rust-{self.current_version}.ebuild',
+ f'rust-{self.new_version}.ebuild'
+ ]
+ actual = rust_uprev.find_oldest_rust_version_in_chroot()
+ expected = (self.old_version,
+ os.path.join(rust_uprev.RUST_PATH, oldest_version_name))
+ self.assertEqual(expected, actual)
+
+ @mock.patch.object(os, 'listdir')
+ def test_find_oldest_rust_version_in_chroot_fail_with_only_one_ebuild(
+ self, mock_ls):
+ mock_ls.return_value = [f'rust-{self.new_version}.ebuild']
+ with self.assertRaises(RuntimeError) as context:
+ rust_uprev.find_oldest_rust_version_in_chroot()
+ self.assertEqual('Expect to find more than one Rust versions',
+ str(context.exception))
+
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ @mock.patch.object(git, 'CreateBranch')
+ def test_create_new_repo(self, mock_branch, mock_output):
+ mock_output.return_value = ''
+ rust_uprev.create_new_repo(self.new_version)
+ mock_branch.assert_called_once_with(rust_uprev.RUST_PATH,
+ f'rust-to-{self.new_version}')
+
+ @mock.patch.object(rust_uprev, 'get_command_output')
+ @mock.patch.object(subprocess, 'check_call')
+ def test_build_cross_compiler(self, mock_call, mock_output):
+ mock_output.return_value = f'rust-{self.new_version}.ebuild'
+ cros_targets = [
+ 'x86_64-cros-linux-gnu',
+ 'armv7a-cros-linux-gnueabihf',
+ 'aarch64-cros-linux-gnu',
+ ]
+ all_triples = ['x86_64-pc-linux-gnu'] + cros_targets
+ rust_ebuild = 'RUSTC_TARGET_TRIPLES=(' + '\n\t'.join(all_triples) + ')'
+ mock_open = mock.mock_open(read_data=rust_ebuild)
+ with mock.patch('builtins.open', mock_open):
+ rust_uprev.build_cross_compiler()
+
+ mock_call.assert_called_once_with(
+ ['sudo', 'emerge', '-j', '-G'] +
+ [f'cross-{x}/gcc' for x in cros_targets + ['arm-none-eabi']])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py
new file mode 100755
index 00000000..b9ad7b82
--- /dev/null
+++ b/rust_tools/rust_watch.py
@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Checks for various upstream events with the Rust toolchain.
+
+Sends an email if something interesting (probably) happened.
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import itertools
+import json
+import logging
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+import time
+from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple
+
+from cros_utils import email_sender
+from cros_utils import tiny_render
+
+
+def gentoo_sha_to_link(sha: str) -> str:
+ """Gets a URL to a webpage that shows the Gentoo commit at `sha`."""
+ return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}'
+
+
+def send_email(subject: str, body: List[tiny_render.Piece]) -> None:
+ """Sends an email with the given title and body to... whoever cares."""
+ email_sender.EmailSender().SendX20Email(
+ subject=subject,
+ identifier='rust-watch',
+ well_known_recipients=['cros-team'],
+ text_body=tiny_render.render_text_pieces(body),
+ html_body=tiny_render.render_html_pieces(body),
+ )
+
+
+class RustReleaseVersion(NamedTuple):
+ """Represents a version of Rust's stable compiler."""
+ major: int
+ minor: int
+ patch: int
+
+ @staticmethod
+ def from_string(version_string: str) -> 'RustReleaseVersion':
+ m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string)
+ if not m:
+ raise ValueError(f"{version_string!r} isn't a valid version string")
+ return RustReleaseVersion(*[int(x) for x in m.groups()])
+
+ def __str__(self) -> str:
+ return f'{self.major}.{self.minor}.{self.patch}'
+
+ def to_json(self) -> str:
+ return str(self)
+
+ @staticmethod
+ def from_json(s: str) -> 'RustReleaseVersion':
+ return RustReleaseVersion.from_string(s)
+
+
+class State(NamedTuple):
+ """State that we keep around from run to run."""
+ # The last Rust release tag that we've seen.
+ last_seen_release: RustReleaseVersion
+
+ # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen
+ # that updates it.
+ last_gentoo_sha: str
+
+ def to_json(self) -> Dict[str, Any]:
+ return {
+ 'last_seen_release': self.last_seen_release.to_json(),
+ 'last_gentoo_sha': self.last_gentoo_sha,
+ }
+
+ @staticmethod
+ def from_json(s: Dict[str, Any]) -> 'State':
+ return State(
+ last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']),
+ last_gentoo_sha=s['last_gentoo_sha'],
+ )
+
+
+def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]:
+ """Parses `git ls-remote --tags` output into Rust stable release versions."""
+ refs_tags = 'refs/tags/'
+ for line in lines:
+ _sha, tag = line.split(None, 1)
+ tag = tag.strip()
+ # Each tag has an associated 'refs/tags/name^{}', which is the actual
+ # object that the tag points to. That's irrelevant to us.
+ if tag.endswith('^{}'):
+ continue
+
+ if not tag.startswith(refs_tags):
+ continue
+
+ short_tag = tag[len(refs_tags):]
+ # There are a few old versioning schemes. Ignore them.
+ if short_tag.startswith('0.') or short_tag.startswith('release-'):
+ continue
+ yield RustReleaseVersion.from_string(short_tag)
+
+
+def fetch_most_recent_release() -> RustReleaseVersion:
+ """Fetches the most recent stable `rustc` version."""
+ result = subprocess.run(
+ ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'],
+ check=True,
+ stdin=None,
+ capture_output=True,
+ encoding='utf-8',
+ )
+ tag_lines = result.stdout.strip().splitlines()
+ return max(parse_release_tags(tag_lines))
+
+
+class GitCommit(NamedTuple):
+ """Represents a single git commit."""
+ sha: str
+ subject: str
+
+
+def update_git_repo(git_dir: pathlib.Path) -> None:
+ """Updates the repo at `git_dir`, retrying a few times on failure."""
+ for i in itertools.count(start=1):
+ result = subprocess.run(
+ ['git', 'fetch', 'origin'],
+ check=False,
+ cwd=str(git_dir),
+ stdin=None,
+ )
+
+ if not result.returncode:
+ break
+
+ if i == 5:
+ # 5 attempts is too many. Something else may be wrong.
+ result.check_returncode()
+
+ sleep_time = 60 * i
+ logging.error("Failed updating gentoo's repo; will try again in %ds...",
+ sleep_time)
+ time.sleep(sleep_time)
+
+
+def get_new_gentoo_commits(git_dir: pathlib.Path,
+ most_recent_sha: str) -> List[GitCommit]:
+ """Gets commits to dev-lang/rust since `most_recent_sha`.
+
+ Older commits come earlier in the returned list.
+ """
+ commits = subprocess.run(
+ [
+ 'git',
+ 'log',
+ '--format=%H %s',
+ f'{most_recent_sha}..origin/master',
+ '--',
+ 'dev-lang/rust',
+ ],
+ capture_output=True,
+ check=False,
+ cwd=str(git_dir),
+ encoding='utf-8',
+ )
+
+ if commits.returncode:
+ logging.error('Error getting new gentoo commits; stderr:\n%s',
+ commits.stderr)
+ commits.check_returncode()
+
+ results = []
+ for line in commits.stdout.strip().splitlines():
+ sha, subject = line.strip().split(None, 1)
+ results.append(GitCommit(sha=sha, subject=subject))
+
+ # `git log` outputs things in newest -> oldest order.
+ results.reverse()
+ return results
+
+
+def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str:
+ """Sets up a gentoo git repo at the given directory. Returns HEAD."""
+ subprocess.run(
+ [
+ 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git',
+ str(git_dir)
+ ],
+ stdin=None,
+ check=True,
+ )
+
+ head_rev = subprocess.run(
+ ['git', 'rev-parse', 'HEAD'],
+ cwd=str(git_dir),
+ check=True,
+ stdin=None,
+ capture_output=True,
+ encoding='utf-8',
+ )
+ return head_rev.stdout.strip()
+
+
+def read_state(state_file: pathlib.Path) -> State:
+ """Reads state from the given file."""
+ with state_file.open(encoding='utf-8') as f:
+ return State.from_json(json.load(f))
+
+
+def atomically_write_state(state_file: pathlib.Path, state: State) -> None:
+ """Writes state to the given file."""
+ temp_file = pathlib.Path(str(state_file) + '.new')
+ with temp_file.open('w', encoding='utf-8') as f:
+ json.dump(state.to_json(), f)
+ temp_file.rename(state_file)
+
+
+def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion,
+ new_gentoo_commits: List[GitCommit]
+ ) -> Optional[Tuple[str, List[tiny_render.Piece]]]:
+ """Creates an email given our new state, if doing so is appropriate."""
+ subject_pieces = []
+ body_pieces = []
+
+ if newest_release > old_state.last_seen_release:
+ subject_pieces.append('new rustc release detected')
+ body_pieces.append(f'Rustc tag for v{newest_release} was found.')
+
+ if new_gentoo_commits:
+ # Separate the sections a bit for prettier output.
+ if body_pieces:
+ body_pieces += [tiny_render.line_break, tiny_render.line_break]
+
+ if len(new_gentoo_commits) == 1:
+ subject_pieces.append('new rust ebuild commit detected')
+ body_pieces.append('commit:')
+ else:
+ subject_pieces.append('new rust ebuild commits detected')
+ body_pieces.append('commits (newest first):')
+
+ commit_lines = []
+ for commit in new_gentoo_commits:
+ commit_lines.append([
+ tiny_render.Link(
+ gentoo_sha_to_link(commit.sha),
+ commit.sha[:12],
+ ),
+ f': {commit.subject}',
+ ])
+
+ body_pieces.append(tiny_render.UnorderedList(commit_lines))
+
+ if not subject_pieces:
+ return None
+
+ subject = '[rust-watch] ' + '; '.join(subject_pieces)
+ return subject, body_pieces
+
+
+def main(argv: List[str]) -> None:
+ logging.basicConfig(level=logging.INFO)
+
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument(
+ '--state_dir', required=True, help='Directory to store state in.')
+ parser.add_argument(
+ '--skip_email', action='store_true', help="Don't send an email.")
+ parser.add_argument(
+ '--skip_state_update',
+ action='store_true',
+ help="Don't update the state file. Doesn't apply to initial setup.")
+ opts = parser.parse_args(argv)
+
+ state_dir = pathlib.Path(opts.state_dir)
+ state_file = state_dir / 'state.json'
+ gentoo_subdir = state_dir / 'upstream-gentoo'
+ if not state_file.exists():
+ logging.info("state_dir isn't fully set up; doing that now.")
+
+ # Could be in a partially set-up state.
+ if state_dir.exists():
+ logging.info('incomplete state_dir detected; removing.')
+ shutil.rmtree(str(state_dir))
+
+ state_dir.mkdir(parents=True)
+ most_recent_release = fetch_most_recent_release()
+ most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir)
+ atomically_write_state(
+ state_file,
+ State(
+ last_seen_release=most_recent_release,
+ last_gentoo_sha=most_recent_gentoo_commit,
+ ),
+ )
+ # Running through this _should_ be a nop, but do it anyway. Should make any
+ # bugs more obvious on the first run of the script.
+
+ prior_state = read_state(state_file)
+ logging.info('Last state was %r', prior_state)
+
+ most_recent_release = fetch_most_recent_release()
+ logging.info('Most recent Rust release is %s', most_recent_release)
+
+ logging.info('Fetching new commits from Gentoo')
+ update_git_repo(gentoo_subdir)
+ new_commits = get_new_gentoo_commits(gentoo_subdir,
+ prior_state.last_gentoo_sha)
+ logging.info('New commits: %r', new_commits)
+
+ maybe_email = maybe_compose_email(prior_state, most_recent_release,
+ new_commits)
+
+ if maybe_email is None:
+ logging.info('No updates to send')
+ else:
+ title, body = maybe_email
+ if opts.skip_email:
+ logging.info('Skipping sending email with title %r and contents\n%s',
+ title, tiny_render.render_html_pieces(body))
+ else:
+ logging.info('Sending email')
+ send_email(title, body)
+
+ if opts.skip_state_update:
+ logging.info('Skipping state update, as requested')
+ return
+
+ newest_sha = (
+ new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha)
+ atomically_write_state(
+ state_file,
+ State(
+ last_seen_release=most_recent_release,
+ last_gentoo_sha=newest_sha,
+ ),
+ )
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py
new file mode 100755
index 00000000..97d111fc
--- /dev/null
+++ b/rust_tools/rust_watch_test.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for rust_watch.py."""
+
+# pylint: disable=cros-logging-import
+
+import logging
+import pathlib
+import subprocess
+import time
+import unittest
+import unittest.mock
+
+import rust_watch
+from cros_utils import tiny_render
+
+
+class Test(unittest.TestCase):
+ """Tests."""
+
+ def _silence_logs(self):
+ """Silences all log output until the end of the current test."""
+
+ def should_log(_record):
+ return 0
+
+ logger = logging.root
+ logger.addFilter(should_log)
+ self.addCleanup(logger.removeFilter, should_log)
+
+ def test_release_version_parsing(self):
+ self.assertEqual(
+ rust_watch.RustReleaseVersion.from_string('1.2.3'),
+ rust_watch.RustReleaseVersion(1, 2, 3),
+ )
+
+ def test_release_version_json_round_trips(self):
+ ver = rust_watch.RustReleaseVersion(1, 2, 3)
+ self.assertEqual(
+ rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver)
+
+ def test_state_json_round_trips(self):
+ state = rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3),
+ last_gentoo_sha='abc123',
+ )
+
+ self.assertEqual(rust_watch.State.from_json(state.to_json()), state)
+
+ @unittest.mock.patch.object(subprocess, 'run')
+ @unittest.mock.patch.object(time, 'sleep')
+ def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock):
+ self._silence_logs()
+
+ oh_no_error = ValueError('oh no')
+
+ def check_returncode():
+ raise oh_no_error
+
+ run_call_count = 0
+
+ def run_sideeffect(*_args, **_kwargs):
+ nonlocal run_call_count
+ run_call_count += 1
+ result = unittest.mock.Mock()
+ result.returncode = 1
+ result.check_returncode = check_returncode
+ return result
+
+ run_mock.side_effect = run_sideeffect
+
+ with self.assertRaises(ValueError) as raised:
+ rust_watch.update_git_repo(pathlib.Path('/does/not/exist/at/all'))
+
+ self.assertIs(raised.exception, oh_no_error)
+ self.assertEqual(run_call_count, 5)
+
+ sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)]
+ self.assertEqual(sleep_mock.mock_calls, sleep_timings)
+
+ @unittest.mock.patch.object(subprocess, 'run')
+ def test_get_new_gentoo_commits_functions(self, run_mock):
+ returned = unittest.mock.Mock()
+ returned.returncode = 0
+ returned.stdout = '\n'.join((
+ 'abc123 newer commit',
+ 'abcdef and an older commit',
+ ))
+ run_mock.return_value = returned
+ results = rust_watch.get_new_gentoo_commits(
+ pathlib.Path('/does/not/exist/at/all'), 'defabc')
+ self.assertEqual(results, [
+ rust_watch.GitCommit('abcdef', 'and an older commit'),
+ rust_watch.GitCommit('abc123', 'newer commit'),
+ ])
+
+ def test_compose_email_on_a_new_release(self):
+ new_release = rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 1, 0),
+ new_gentoo_commits=[],
+ )
+
+ self.assertEqual(new_release, ('[rust-watch] new rustc release detected',
+ ['Rustc tag for v1.1.0 was found.']))
+
+ def test_compose_email_on_a_new_gentoo_commit(self):
+ sha_a = 'a' * 40
+ new_commit = rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ new_gentoo_commits=[
+ rust_watch.GitCommit(
+ sha=sha_a,
+ subject='summary_a',
+ ),
+ ],
+ )
+
+ self.assertEqual(new_commit,
+ ('[rust-watch] new rust ebuild commit detected', [
+ 'commit:',
+ tiny_render.UnorderedList([
+ [
+ tiny_render.Link(
+ rust_watch.gentoo_sha_to_link(sha_a),
+ sha_a[:12],
+ ),
+ ': summary_a',
+ ],
+ ])
+ ]))
+
+ def test_compose_email_on_multiple_events(self):
+ sha_a = 'a' * 40
+ new_commit_and_release = rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 1, 0),
+ new_gentoo_commits=[
+ rust_watch.GitCommit(
+ sha=sha_a,
+ subject='summary_a',
+ ),
+ ],
+ )
+
+ self.assertEqual(
+ new_commit_and_release,
+ ('[rust-watch] new rustc release detected; new rust ebuild commit '
+ 'detected', [
+ 'Rustc tag for v1.1.0 was found.',
+ tiny_render.line_break,
+ tiny_render.line_break,
+ 'commit:',
+ tiny_render.UnorderedList([
+ [
+ tiny_render.Link(
+ rust_watch.gentoo_sha_to_link(sha_a),
+ sha_a[:12],
+ ),
+ ': summary_a',
+ ],
+ ]),
+ ]))
+
+ def test_compose_email_composes_nothing_when_no_new_updates_exist(self):
+ self.assertIsNone(
+ rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ new_gentoo_commits=[],
+ ))
+
+ self.assertIsNone(
+ rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ new_gentoo_commits=[],
+ ))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/setup_chromeos.py b/setup_chromeos.py
index 63206492..07c7530d 100755
--- a/setup_chromeos.py
+++ b/setup_chromeos.py
@@ -244,7 +244,7 @@ Default is 'latest_lkgm'.""")
return cmd_executer.RunCommand(
'git ls-remote '
'https://chrome-internal.googlesource.com/chrome/src-internal.git '
- '> /dev/null')
+ 'refs/HEAD > /dev/null')
if __name__ == '__main__':
diff --git a/toolchain_utils_githooks/check-presubmit.py b/toolchain_utils_githooks/check-presubmit.py
index 26373342..274a75f4 100755
--- a/toolchain_utils_githooks/check-presubmit.py
+++ b/toolchain_utils_githooks/check-presubmit.py
@@ -7,11 +7,7 @@
"""Runs presubmit checks against a bundle of files."""
-# To keep `cros lint` happy
-from __future__ import division, print_function
-
import argparse
-import collections
import datetime
import multiprocessing
import multiprocessing.pool
@@ -26,7 +22,9 @@ import traceback
import typing as t
-def run_command_unchecked(command, cwd, env=None):
+def run_command_unchecked(command: t.List[str],
+ cwd: str,
+ env: t.Dict[str, str] = None) -> t.Tuple[int, str]:
"""Runs a command in the given dir, returning its exit code and stdio."""
p = subprocess.Popen(
command,
@@ -42,12 +40,12 @@ def run_command_unchecked(command, cwd, env=None):
return exit_code, stdout.decode('utf-8', 'replace')
-def has_executable_on_path(exe):
+def has_executable_on_path(exe: str) -> bool:
"""Returns whether we have `exe` somewhere on our $PATH"""
return shutil.which(exe) is not None
-def escape_command(command):
+def escape_command(command: t.Iterable[str]) -> str:
"""Returns a human-readable and copy-pastable shell command.
Only intended for use in output to users. shell=True is strongly discouraged.
@@ -55,18 +53,18 @@ def escape_command(command):
return ' '.join(shlex.quote(x) for x in command)
-def remove_deleted_files(files):
+def remove_deleted_files(files: t.Iterable[str]) -> t.List[str]:
return [f for f in files if os.path.exists(f)]
-def is_file_executable(file_path):
+def is_file_executable(file_path: str) -> bool:
return os.access(file_path, os.X_OK)
# As noted in our docs, some of our Python code depends on modules that sit in
# toolchain-utils/. Add that to PYTHONPATH to ensure that things like `cros
# lint` are kept happy.
-def env_with_pythonpath(toolchain_utils_root):
+def env_with_pythonpath(toolchain_utils_root: str) -> t.Dict[str, str]:
env = dict(os.environ)
if 'PYTHONPATH' in env:
env['PYTHONPATH'] += ':' + toolchain_utils_root
@@ -86,12 +84,22 @@ def env_with_pythonpath(toolchain_utils_root):
# least ${number_of_concurrently_running_checkers}+1 threads are present
# in the pool. In order words, blocking on results from the provided
# threadpool is OK.
-CheckResult = collections.namedtuple('CheckResult',
- ('ok', 'output', 'autofix_commands'))
+CheckResult = t.NamedTuple(
+ 'CheckResult',
+ (
+ ('ok', bool),
+ ('output', str),
+ ('autofix_commands', t.List[t.List[str]]),
+ ),
+)
+
+def get_check_result_or_catch(
+ task: multiprocessing.pool.ApplyResult) -> CheckResult:
+ """Returns the result of task(); if that raises, returns a CheckResult.
-def get_check_result_or_catch(task):
- """Returns the result of task(); if that raises, returns a CheckResult."""
+ The task is expected to return a CheckResult on get().
+ """
try:
return task.get()
except Exception:
@@ -103,7 +111,8 @@ def get_check_result_or_catch(task):
)
-def check_yapf(toolchain_utils_root, python_files):
+def check_yapf(toolchain_utils_root: str,
+ python_files: t.Iterable[str]) -> CheckResult:
"""Subchecker of check_py_format. Checks python file formats with yapf"""
command = ['yapf', '-d'] + python_files
exit_code, stdout_and_stderr = run_command_unchecked(
@@ -146,7 +155,7 @@ def check_yapf(toolchain_utils_root, python_files):
)
-def check_python_file_headers(python_files):
+def check_python_file_headers(python_files: t.Iterable[str]) -> CheckResult:
"""Subchecker of check_py_format. Checks python #!s"""
add_hashbang = []
remove_hashbang = []
@@ -188,7 +197,9 @@ def check_python_file_headers(python_files):
)
-def check_py_format(toolchain_utils_root, thread_pool, files):
+def check_py_format(toolchain_utils_root: str,
+ thread_pool: multiprocessing.pool.ThreadPool,
+ files: t.Iterable[str]) -> CheckResult:
"""Runs yapf on files to check for style bugs. Also checks for #!s."""
yapf = 'yapf'
if not has_executable_on_path(yapf):
@@ -217,7 +228,13 @@ def check_py_format(toolchain_utils_root, thread_pool, files):
return [(name, get_check_result_or_catch(task)) for name, task in tasks]
-def check_cros_lint(toolchain_utils_root, thread_pool, files):
+def find_chromeos_root_directory() -> t.Optional[str]:
+ return os.getenv('CHROMEOS_ROOT_DIRECTORY')
+
+
+def check_cros_lint(
+ toolchain_utils_root: str, thread_pool: multiprocessing.pool.ThreadPool,
+ files: t.Iterable[str]) -> t.Union[t.List[CheckResult], CheckResult]:
"""Runs `cros lint`"""
fixed_env = env_with_pythonpath(toolchain_utils_root)
@@ -225,7 +242,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files):
# We have to support users who don't have a chroot. So we either run `cros
# lint` (if it's been made available to us), or we try a mix of
# pylint+golint.
- def try_run_cros_lint(cros_binary):
+ def try_run_cros_lint(cros_binary: str) -> t.Optional[CheckResult]:
exit_code, output = run_command_unchecked(
[cros_binary, 'lint', '--py3', '--'] + files,
toolchain_utils_root,
@@ -246,7 +263,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files):
if cros_lint is not None:
return cros_lint
- cros_root = os.getenv('CHROMEOS_ROOT_DIRECTORY')
+ cros_root = find_chromeos_root_directory()
if cros_root:
cros_lint = try_run_cros_lint(os.path.join(cros_root, 'chromite/bin/cros'))
if cros_lint is not None:
@@ -254,7 +271,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files):
tasks = []
- def check_result_from_command(command):
+ def check_result_from_command(command: t.List[str]) -> CheckResult:
exit_code, output = run_command_unchecked(
command, toolchain_utils_root, env=fixed_env)
return CheckResult(
@@ -266,7 +283,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files):
python_files = [f for f in remove_deleted_files(files) if f.endswith('.py')]
if python_files:
- def run_pylint():
+ def run_pylint() -> CheckResult:
# pylint is required. Fail hard if it DNE.
return check_result_from_command(['pylint'] + python_files)
@@ -275,7 +292,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files):
go_files = [f for f in remove_deleted_files(files) if f.endswith('.go')]
if go_files:
- def run_golint():
+ def run_golint() -> CheckResult:
if has_executable_on_path('golint'):
return check_result_from_command(['golint', '-set_exit_status'] +
go_files)
@@ -362,7 +379,9 @@ def check_go_format(toolchain_utils_root, _thread_pool, files):
)
-def check_tests(toolchain_utils_root, _thread_pool, files):
+def check_tests(toolchain_utils_root: str,
+ _thread_pool: multiprocessing.pool.ThreadPool,
+ files: t.List[str]) -> CheckResult:
"""Runs tests."""
exit_code, stdout_and_stderr = run_command_unchecked(
[os.path.join(toolchain_utils_root, 'run_tests_for.py'), '--'] + files,
@@ -374,15 +393,17 @@ def check_tests(toolchain_utils_root, _thread_pool, files):
)
-def detect_toolchain_utils_root():
+def detect_toolchain_utils_root() -> str:
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-def process_check_result(check_name, check_results, start_time):
+def process_check_result(
+ check_name: str, check_results: t.Union[t.List[CheckResult], CheckResult],
+ start_time: datetime.datetime) -> t.Tuple[bool, t.List[t.List[str]]]:
"""Prints human-readable output for the given check_results."""
indent = ' '
- def indent_block(text):
+ def indent_block(text: str) -> str:
return indent + text.replace('\n', '\n' + indent)
if isinstance(check_results, CheckResult):
@@ -426,7 +447,8 @@ def process_check_result(check_name, check_results, start_time):
return ok, autofix_commands
-def try_autofix(all_autofix_commands, toolchain_utils_root):
+def try_autofix(all_autofix_commands: t.List[t.List[str]],
+ toolchain_utils_root: str) -> None:
"""Tries to run all given autofix commands, if appropriate."""
if not all_autofix_commands:
return
@@ -478,10 +500,16 @@ def maybe_reexec_inside_chroot(autofix: bool, files: t.List[str]) -> None:
return
enter_chroot = True
+ chdir_to = None
toolchain_utils = detect_toolchain_utils_root()
if find_repo_root(toolchain_utils) is None:
- print('Standalone toolchain-utils checkout detected; cannot enter chroot.')
- enter_chroot = False
+ chromeos_root_dir = find_chromeos_root_directory()
+ if chromeos_root_dir is None:
+ print('Standalone toolchain-utils checkout detected; cannot enter '
+ 'chroot.')
+ enter_chroot = False
+ else:
+ chdir_to = chromeos_root_dir
if not has_executable_on_path('cros_sdk'):
print('No `cros_sdk` detected on $PATH; cannot enter chroot.')
@@ -511,7 +539,11 @@ def maybe_reexec_inside_chroot(autofix: bool, files: t.List[str]) -> None:
args.append('--no_autofix')
args.extend(rebase_path(x) for x in files)
- print('Attempting to enter the chroot...')
+ if chdir_to is None:
+ print('Attempting to enter the chroot...')
+ else:
+ print(f'Attempting to enter the chroot for tree at {chdir_to}...')
+ os.chdir(chdir_to)
os.execvp(args[0], args)
@@ -534,7 +566,7 @@ def ensure_scipy_installed() -> None:
subprocess.check_call(['pip', 'install', '--user', 'scipy'])
-def main(argv):
+def main(argv: t.List[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--no_autofix',
diff --git a/unblocked_terms.txt b/unblocked_terms.txt
new file mode 100644
index 00000000..c961e667
--- /dev/null
+++ b/unblocked_terms.txt
@@ -0,0 +1,5 @@
+# Code uses this in many cases to refer to git branches.
+master
+
+# A few paths in AOSP contain this term.
+native