diff options
author | Pirama Arumuga Nainar <pirama@google.com> | 2020-10-22 08:24:15 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2020-10-22 08:24:15 +0000 |
commit | 7a702fb88be5cf729d1a248d049e50ca1d35ed16 (patch) | |
tree | 073e9e403bc6e2290b2b5b005a5cfdabadde1e42 | |
parent | bdd29acf271dfaee411a6e3434c0d2bd612834a1 (diff) | |
parent | 850d25c38cd14586fd263215480c3176fbd36480 (diff) | |
download | toolchain-utils-7a702fb88be5cf729d1a248d049e50ca1d35ed16.tar.gz |
Merging 84 commit(s) from Chromium's toolchain-utils am: 1e576757bc am: 12fcdf982e am: 76b8300539 am: 850d25c38c
Original change: https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/1469797
Change-Id: I0073071a09fd8710738e053a8726e21bfba0b1c8
106 files changed, 5240 insertions, 2719 deletions
diff --git a/OWNERS.toolchain b/OWNERS.toolchain index f7d6e3f6..48ecc260 100644 --- a/OWNERS.toolchain +++ b/OWNERS.toolchain @@ -1,10 +1,10 @@ cmtice@chromium.org denik@chromium.org gbiv@chromium.org +inglorion@chromium.org jiancai@chromium.org llozano@chromium.org manojgupta@chromium.org -tbosch@chromium.org tcwang@chromium.org zhizhouy@chromium.org diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index a6f289f2..2f4ead4a 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,14 @@ { - "chromeos-kernel-4_4": { - "name": "R85-13280.0-1592213800" - }, "chromeos-kernel-3_18": { - "name": "R85-13280.0-1592213569" + "name": "R88-13502.0-1602497202" + }, + "chromeos-kernel-4_4": { + "name": "R88-13505.4-1602496005" }, "chromeos-kernel-4_14": { - "name": "R85-13280.0-1592214019" + "name": "R88-13505.4-1602496823" }, "chromeos-kernel-4_19": { - "name": "R85-13280.0-1592214191" + "name": "R88-13502.0-1602496205" } } diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo new file mode 100755 index 00000000..8eb1306c --- /dev/null +++ b/afdo_tools/update_kernel_afdo @@ -0,0 +1,118 @@ +#!/bin/bash +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Due to crbug.com/1081332, we need to update AFDO metadata +# manually. This script performs a few checks and generates a +# new kernel_afdo.json file, which can then be committed. +# +# USAGE: +# toolchain-utils$ ./afdo_tools/update_kernel_afdo +# +# The script modifies the JSON file and shows the git diff. +# +# If the changes look good, git commit them. Example commit +# message (from crrev.com/c/2197462): +# +# afdo_metadata: Publish the new kernel profiles +# +# Update chromeos-kernel-3_18 to R84-13080.0-1589189810 +# Update chromeos-kernel-4_4 to R84-13080.0-1589189726 +# Update chromeos-kernel-4_14 to R84-13080.0-1589190025 +# Update chromeos-kernel-4_19 to R84-13080.0-1589189550 +# +# BUG=None +# TEST=Verified in kernel-release-afdo-verify-orchestrator. +# + +set -eu +set -o pipefail + +CROS_REPO=https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay +GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel +KVERS="3.18 4.4 4.14 4.19" +errs="" +successes=0 + +script_dir=$(dirname "$0") +tc_utils_dir="$script_dir/.." +metadata_dir="$tc_utils_dir/afdo_metadata" +outfile="$metadata_dir/kernel_afdo.json" + +# The most recent Monday, in Unix timestamp format. +if [ $(date +%a) = "Mon" ] +then + expected_time=$(date +%s -d 00:00:00) +else + expected_time=$(date +%s -d "last Monday") +fi + +# Get the current canary branch number (using beta + 1) +beta=$(git ls-remote -h $CROS_REPO | \ + sed -n -e "s/^.*release-R\([0-9][0-9]*\).*$/\1/p" | \ + sort -g | tail -1) +canary="$(($beta + 1))" + +json="{" +sep="" +for kver in $KVERS +do + # Sort the gs output by timestamp (default ordering is by name, so + # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) + latest=$(gsutil ls -l "$GS_BASE/$kver/" | sort -k2 | \ + grep "R${canary}" | tail -1 || true) + if [ -z "$latest" ] + then + # if no profiles exist for R${canary}, try the previous branch + latest=$(gsutil ls -l "$GS_BASE/$kver/" | sort -k2 | \ + grep "R${beta}" | tail -1) + fi + + # Verify that the file has the expected date. + file_time=$(echo "$latest" | awk '{print $2}') + file_time_unix=$(date +%s -d "$file_time") + if [ $file_time_unix -lt $expected_time ] + then + expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) + echo "Wrong date for $kver: $file_time is before $expected" >&2 + errs="$errs $kver" + continue + fi + + # Generate JSON. + json_kver=$(echo "$kver" | tr . _) + # b/147370213 (migrating profiles from gcov format) may result in the + # pattern below no longer doing the right thing. + name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%') + json=$(cat <<EOT +$json$sep + "chromeos-kernel-$json_kver": { + "name": "$name" + } +EOT +) + sep="," + successes=$((successes + 1)) +done + +# If we did not succeed for any kvers, exit now. +if [ $successes -eq 0 ] +then + echo "error: AFDO profiles out of date for all kernel versions" >&2 + exit 2 +fi + +# Write new JSON file. +printf "%s\n}\n" "$json" > "$outfile" + +# Show the changes. +(cd "$tc_utils_dir" && git diff) + +# If no changes were made, say so. +outdir=$(dirname "$outfile") +shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) +[ -n "$shortstat" ] || echo $(basename "$outfile")" is up to date." + +# If we had any errors, warn about them. +[ -z "$errs" ] || echo "warning: failed to update$errs" >&2 diff --git a/android_bench_suite/fix_skia_results.py b/android_bench_suite/fix_skia_results.py index 8c919d35..84dee5a5 100755 --- a/android_bench_suite/fix_skia_results.py +++ b/android_bench_suite/fix_skia_results.py @@ -5,6 +5,7 @@ # found in the LICENSE file. # # pylint: disable=cros-logging-import + """Transforms skia benchmark results to ones that crosperf can understand.""" from __future__ import print_function @@ -57,7 +58,10 @@ def _GetTimeMultiplier(label_name): def _GetTimeDenom(ms): - """Given a list of times (in milliseconds), find a sane time unit for them. + """Express times in a common time unit. + + Given a list of times (in milliseconds), find a time unit in which + they can all be expressed. Returns the unit name, and `ms` normalized to that time unit. @@ -94,9 +98,9 @@ def _TransformBenchmarks(raw_benchmarks): # statistic... benchmarks = raw_benchmarks['results'] results = [] - for bench_name, bench_result in benchmarks.iteritems(): + for bench_name, bench_result in benchmarks.items(): try: - for cfg_name, keyvals in bench_result.iteritems(): + for cfg_name, keyvals in bench_result.items(): # Some benchmarks won't have timing data (either it won't exist at all, # or it'll be empty); skip them. samples = keyvals.get('samples') @@ -109,17 +113,16 @@ def _TransformBenchmarks(raw_benchmarks): friendly_name = _GetFamiliarName(bench_name) if len(results) < len(samples): - results.extend({ - 'retval': 0 - } for _ in range(len(samples) - len(results))) + results.extend( + {'retval': 0} for _ in range(len(samples) - len(results))) time_mul = _GetTimeMultiplier(friendly_name) - for sample, app in itertools.izip(samples, results): + for sample, app in itertools.zip(samples, results): assert friendly_name not in app app[friendly_name] = sample * time_mul except (KeyError, ValueError) as e: - logging.error('While converting "%s" (key: %s): %s', - bench_result, bench_name, e.message) + logging.error('While converting "%s" (key: %s): %s', bench_result, + bench_name, e) raise # Realistically, [results] should be multiple results, where each entry in the diff --git a/android_merge_from_upstream.sh b/android_merge_from_upstream.sh index cf07d4bf..b63eb6e1 100755 --- a/android_merge_from_upstream.sh +++ b/android_merge_from_upstream.sh @@ -18,11 +18,11 @@ remote_branch="${remote}/upstream-mirror-master" my_dir="$(dirname "$(readlink -m "$0")")" cd "${my_dir}" -ensure_head_is_upstream_master() { - local current_rev master_rev +ensure_head_is_upstream_main() { + local current_rev main_rev current_rev="$(git rev-parse HEAD)" - master_rev="$(git rev-parse ${local_upstream})" - if [[ "${current_rev}" != "${master_rev}" ]]; then + main_rev="$(git rev-parse ${local_upstream})" + if [[ "${current_rev}" != "${main_rev}" ]]; then echo "Please checkout ${local_upstream} and rerun this" >&2 exit fi @@ -42,16 +42,8 @@ ensure_no_local_branch_present() { exit 1 fi - # If we're *on* that branch, deleting it is difficult. - local current_branch - current_branch="$(git branch --show-current)" - if [[ "${current_branch}" == "${local_branch_name}" ]]; then - local rev - rev="$(git rev-parse HEAD)" - # This is fine, since we assume HEAD == upstream-mirror-master anyway - # (e.g., the existing branch was pointless.) - git checkout "${rev}" - fi + # If we're *on* that branch, deleting it is difficult. Always detach. + git checkout --detach || return git branch -D "${local_branch_name}" } @@ -61,7 +53,7 @@ get_merge_commit_list() { git log --oneline "${merge_base}..${remote_branch}" } -ensure_head_is_upstream_master +ensure_head_is_upstream_main ensure_no_local_branch_present echo "Ensuring repository is up-to-date..." diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py index e40b95c2..c3c2e24c 100755 --- a/auto_delete_nightly_test_data.py +++ b/auto_delete_nightly_test_data.py @@ -16,6 +16,7 @@ import datetime import os import re import shutil +import shlex import sys import time @@ -49,10 +50,10 @@ def CleanNumberedDir(s, dry_run=False): return False ## Now delete the numbered dir Before forcibly removing the directory, just - ## check 's' to make sure it is sane. A valid dir to be removed must be - ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. - valid_dir_pattern = ( - '^' + NIGHTLY_TESTS_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')') + ## check 's' to make sure it matches the expected pattern. A valid dir to be + ## removed must be '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. + valid_dir_pattern = ('^' + NIGHTLY_TESTS_WORKSPACE + '/(' + + '|'.join(DIR_BY_WEEKDAY) + ')') if not re.search(valid_dir_pattern, s): print('Trying to delete an invalid dir "{0}" (must match "{1}"), ' 'please check.'.format(s, valid_dir_pattern)) @@ -192,8 +193,8 @@ def CleanOldCLs(days_to_preserve='1', dry_run=False): ce = command_executer.GetCommandExecuter() chromeos_root = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos') # Find Old CLs. - old_cls_cmd = ( - 'gerrit --raw search "owner:me status:open age:%sd"' % days_to_preserve) + old_cls_cmd = ('gerrit --raw search "owner:me status:open age:%sd"' % + days_to_preserve) _, cls, _ = ce.ChrootRunCommandWOutput( chromeos_root, old_cls_cmd, print_to_console=False) # Convert any whitespaces to spaces. @@ -210,6 +211,25 @@ def CleanOldCLs(days_to_preserve='1', dry_run=False): chromeos_root, abandon_cls_cmd, print_to_console=False) +def CleanChromeTelemetryTmpFiles(dry_run): + rv = 0 + ce = command_executer.GetCommandExecuter() + tmp_dir = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', '.cache', + 'distfiles', 'target', 'chrome-src-internal', 'src', + 'tmp') + cmd = f'rm -fr {shlex.quote(tmp_dir)}/tmp*telemetry_Crosperf' + if dry_run: + print(f'Going to execute:\n{cmd}') + else: + rv = ce.RunCommand(cmd, print_to_console=False) + if rv == 0: + print(f'Successfully cleaned chrome tree tmp directory ' f'{tmp_dir!r} .') + else: + print(f'Some directories were not removed under chrome tree ' + f'tmp directory {tmp_dir!r}.') + return rv + + def Main(argv): """Delete nightly test data directories, tmps and test images.""" options = ProcessArguments(argv) @@ -239,7 +259,10 @@ def Main(argv): # Clean CLs that are not updated in last 2 weeks. rv3 = CleanOldCLs('14', options.dry_run) - return rv + rv2 + rv3 + # Clean telemetry temporaries from chrome source tree inside chroot. + rv4 = CleanChromeTelemetryTmpFiles(options.dry_run) + + return rv + rv2 + rv3 + rv4 if __name__ == '__main__': diff --git a/binary_search_tool/common/test_setup.sh b/binary_search_tool/common/test_setup.sh index 56b8944a..4fa1abe2 100755 --- a/binary_search_tool/common/test_setup.sh +++ b/binary_search_tool/common/test_setup.sh @@ -109,15 +109,17 @@ if [[ "${BISECT_MODE}" == "OBJECT_MODE" ]]; then echo echo "DEPLOYING" - if [[ ${PACKAGE} == sys-kernel/chromeos-kernel-* ]]; then - echo "/mnt/host/source/src/scripts/update_kernel.sh \ - --remote=${BISECT_REMOTE}" - /mnt/host/source/src/scripts/update_kernel.sh --remote=${BISECT_REMOTE} - else - echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}" - cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info + + if [[ ${BISECT_PACKAGE} == sys-kernel/chromeos-kernel-* ]]; then + echo "/mnt/host/source/src/scripts/update_kernel.sh " \ + "--remote=${BISECT_REMOTE} --board=${BISECT_BOARD}" + # exec the command to make sure it always exit after + exec /mnt/host/source/src/scripts/update_kernel.sh --remote=${BISECT_REMOTE} --board=${BISECT_BOARD} fi + echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}" + cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info + deploy_status=$? if [[ ${deploy_status} -eq 0 ]] ; then diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py index e4784a62..48987966 100755 --- a/buildbot_test_toolchains.py +++ b/buildbot_test_toolchains.py @@ -250,6 +250,8 @@ class ToolchainComparator(object): self._ce.RunCommand(command) def _CopyJson(self): + # Make sure a destination directory exists. + os.makedirs(PENDING_ARCHIVES_DIR, exist_ok=True) # Copy json report to pending archives directory. command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) ret = self._ce.RunCommand(command) diff --git a/compiler_wrapper/README.md b/compiler_wrapper/README.md index 794e1635..e55bb669 100644 --- a/compiler_wrapper/README.md +++ b/compiler_wrapper/README.md @@ -12,16 +12,63 @@ package, including the build script, and then build from there without a dependency on toolchain-utils itself. -## Update Chrome OS +## Testing Inside the Chroot -Copy over sources and `build.py` to Chrome OS: +To test updates to the wrapper locally: + +Run `install_compiler_wrapper.sh` to install the new wrapper in the chroot: +``` +(chroot) ~/trunk/src/third_party/toolchain-utils/compiler_wrapper/install_compiler_wrapper.sh +``` + +Then perform the tests, e.g. build with the new compiler. + + +## Updating the Wrapper for Chrome OS + +To update the wrapper for everyone, the new wrapper configuration must be copied +into chromiumos-overlay, and new revisions of the gcc and llvm ebuilds must be +created. + +Copy over sources and `build.py` to chromiumos-overlay: ``` (chroot) /mnt/host/source/src/third_party/chromiumos-overlay/sys-devel/llvm/files/update_compiler_wrapper.sh ``` +Rename chromiumos-overlay/sys-devel/llvm/llvm-${VERSION}.ebuild to the next +revision number. For example, if the current version is +11.0_pre394483_p20200618-r2: +``` +(chroot) cd ~/trunk/src/third_party/chromiumos-overlay +(chroot) git mv llvm-11.0_pre394483_p20200618-r2.ebuild llvm-11.0_pre394483_p20200618-r3.ebuild +``` + +Rename chromiumos-overlay/sys-devel/gcc/gcc-${VERSION}.ebuild to the next +revision number. For example, if the current version is 4.9.2-r254: +``` +(chroot) cd ~/trunk/src/third_party/chromiumos-overlay +(chroot) git mv sys-devel/gcc/gcc-4.9.2-r254.ebuild sys-devel/gcc/gcc-4.9.2-r255.ebuild +``` + +Edit the new ebuild file and change the number at the end to match the new revision: +``` +-# If you need to force a cros_workon uprev, change this number (you can use next +-# uprev): 254 ++# If you need to force a cros_workon uprev, change this number (you can use next ++# uprev): 255 +``` + +Commit those changes together with the changes made by +`update_compiler_wrapper.sh`. + +The changes can then be reviewed and submitted through the normal process. + + +## Paths + `build.py` is called by these ebuilds: -- third_party/chromiumos-overlay/sys-devel/llvm/llvm-9.0_pre361749_p20190714.ebuild +- third_party/chromiumos-overlay/sys-devel/llvm/llvm-*.ebuild - third_party/chromiumos-overlay/sys-devel/gcc/gcc-*.ebuild Generated wrappers are stored here: diff --git a/compiler_wrapper/bisect_flag.go b/compiler_wrapper/bisect_flag.go index f07d9a63..adfa8b03 100644 --- a/compiler_wrapper/bisect_flag.go +++ b/compiler_wrapper/bisect_flag.go @@ -7,6 +7,7 @@ package main import ( "errors" "os" + "os/exec" "path/filepath" ) @@ -52,7 +53,7 @@ func calcBisectCommand(env env, cfg *config, bisectStage string, compilerCmd *co } } absCompilerPath := getAbsCmdPath(env, compilerCmd) - pythonPath, err := filepath.Abs(os.Args[0]) + pythonPath, err := exec.LookPath(os.Args[0]) if err != nil { return nil, err } diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go index 1f0e2231..9eb951de 100644 --- a/compiler_wrapper/clang_flags.go +++ b/compiler_wrapper/clang_flags.go @@ -47,15 +47,7 @@ func processClangFlags(builder *commandBuilder) error { // Use of -Qunused-arguments allows this set to be small, just those // that clang still warns about. unsupported := map[string]bool{ - "-mno-movbe": true, - "-pass-exit-codes": true, - "-Wclobbered": true, - "-Wno-psabi": true, - "-Wlogical-op": true, - "-Wmissing-parameter-type": true, - "-Wold-style-declaration": true, - "-Woverride-init": true, - "-Wunsafe-loop-optimizations": true, + "-pass-exit-codes": true, } unsupportedPrefixes := []string{"-Wstrict-aliasing=", "-finline-limit="} diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go index 2a7fbd15..a4145c22 100644 --- a/compiler_wrapper/clang_flags_test.go +++ b/compiler_wrapper/clang_flags_test.go @@ -212,14 +212,6 @@ func TestFilterUnsupportedClangFlags(t *testing.T) { expectedCount int }{ {clangX86_64, "-pass-exit-codes", 0}, - {clangX86_64, "-Wclobbered", 0}, - {clangX86_64, "-Wunsafe-loop-optimizations", 0}, - {clangX86_64, "-Wlogical-op", 0}, - {clangX86_64, "-Wmissing-parameter-type", 0}, - {clangX86_64, "-Woverride-init", 0}, - {clangX86_64, "-Wold-style-declaration", 0}, - {clangX86_64, "-Wno-psabi", 0}, - {clangX86_64, "-mno-movbe", 0}, {clangX86_64, "-Wstrict-aliasing=xyz", 0}, {clangX86_64, "-finline-limit=xyz", 0}, {"./armv7a-cros-linux-gnu-clang", "-ftrapv", 0}, diff --git a/compiler_wrapper/clang_tidy_flag.go b/compiler_wrapper/clang_tidy_flag.go index d8bf3cb2..01387fd6 100644 --- a/compiler_wrapper/clang_tidy_flag.go +++ b/compiler_wrapper/clang_tidy_flag.go @@ -5,15 +5,39 @@ package main import ( + "encoding/json" "fmt" + "io/ioutil" + "os" + "path" "path/filepath" "strings" ) -func processClangTidyFlags(builder *commandBuilder) (cSrcFile string, useClangTidy bool) { +type useTidyMode int + +const clangTidyCrashSubstring = "PLEASE submit a bug report" + +const ( + tidyModeNone useTidyMode = iota + tidyModeAll + tidyModeTricium +) + +func processClangTidyFlags(builder *commandBuilder) (cSrcFile string, clangTidyFlags []string, mode useTidyMode) { + builder.transformArgs(func(arg builderArg) string { + const prefix = "-clang-tidy-flag=" + if !strings.HasPrefix(arg.value, prefix) { + return arg.value + } + + clangTidyFlags = append(clangTidyFlags, arg.value[len(prefix):]) + return "" + }) + withTidy, _ := builder.env.getenv("WITH_TIDY") if withTidy == "" { - return "", false + return "", clangTidyFlags, tidyModeNone } srcFileSuffixes := []string{ ".c", @@ -24,48 +48,177 @@ func processClangTidyFlags(builder *commandBuilder) (cSrcFile string, useClangTi ".c++", } cSrcFile = "" + srcSuffix := "" lastArg := "" for _, arg := range builder.args { - if hasAtLeastOneSuffix(arg.value, srcFileSuffixes) && lastArg != "-o" { - cSrcFile = arg.value + if lastArg != "-o" { + for _, suffix := range srcFileSuffixes { + if strings.HasSuffix(arg.value, suffix) { + srcSuffix = suffix + cSrcFile = arg.value + break + } + } } lastArg = arg.value } - useClangTidy = cSrcFile != "" - return cSrcFile, useClangTidy -} -func runClangTidy(env env, clangCmd *command, cSrcFile string) error { - defaultTidyChecks := strings.Join([]string{ - "*", - "-bugprone-narrowing-conversions", - "-cppcoreguidelines-*", - "-fuchsia-*", - "-google-readability*", - "-google-runtime-references", - "-hicpp-*", - "-llvm-*", - "-misc-non-private-member-variables-in-classes", - "-misc-unused-parameters", - "-modernize-*", - "-readability-*", - }, ",") + if cSrcFile == "" { + return "", clangTidyFlags, tidyModeNone + } + if withTidy == "tricium" { + // Files generated from protobufs can result in _many_ clang-tidy complaints, and aren't + // worth linting in general. Don't. + if strings.HasSuffix(cSrcFile, ".pb"+srcSuffix) { + mode = tidyModeNone + } else { + mode = tidyModeTricium + } + } else { + mode = tidyModeAll + } + return cSrcFile, clangTidyFlags, mode +} + +func calcClangTidyInvocation(env env, clangCmd *command, cSrcFile string, tidyFlags ...string) (*command, error) { resourceDir, err := getClangResourceDir(env, clangCmd.Path) if err != nil { - return err + return nil, err } clangTidyPath := filepath.Join(filepath.Dir(clangCmd.Path), "clang-tidy") - clangTidyCmd := &command{ - Path: clangTidyPath, - Args: append([]string{ - "-checks=" + defaultTidyChecks, - cSrcFile, - "--", - "-resource-dir=" + resourceDir, - }, clangCmd.Args...), + args := append([]string{}, tidyFlags...) + args = append(args, cSrcFile, "--", "-resource-dir="+resourceDir) + args = append(args, clangCmd.Args...) + return &command{ + Path: clangTidyPath, + Args: args, EnvUpdates: clangCmd.EnvUpdates, + }, nil +} + +func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir string, extraTidyFlags []string, crashArtifactsDir string) error { + if err := os.MkdirAll(fixesDir, 0777); err != nil { + return fmt.Errorf("creating fixes directory at %q: %v", fixesDir, err) + } + + f, err := ioutil.TempFile(fixesDir, "lints-") + if err != nil { + return fmt.Errorf("making tempfile for tidy: %v", err) + } + f.Close() + + // `f` is an 'anchor'; it ensures we won't create a similarly-named file in the future. + // Hence, we can't delete it. + fixesFilePath := f.Name() + ".yaml" + fixesMetadataPath := f.Name() + ".json" + + // FIXME(gbiv): Remove `-checks=*` when testing is complete; we should defer to .clang-tidy + // files, which are both more expressive and more approachable than `-checks=*`. + extraTidyFlags = append(extraTidyFlags, "-checks=*", "--export-fixes="+fixesFilePath) + clangTidyCmd, err := calcClangTidyInvocation(env, clangCmd, cSrcFile, extraTidyFlags...) + if err != nil { + return fmt.Errorf("calculating tidy invocation: %v", err) + } + + stdstreams := &strings.Builder{} + // Note: We pass nil as stdin as we checked before that the compiler + // was invoked with a source file argument. + exitCode, err := wrapSubprocessErrorWithSourceLoc(clangTidyCmd, + env.run(clangTidyCmd, nil, stdstreams, stdstreams)) + if err != nil { + return err + } + + type crashOutput struct { + CrashReproducerPath string `json:"crash_reproducer_path"` + Stdstreams string `json:"stdstreams"` + } + + type metadata struct { + Args []string `json:"args"` + CrashOutput *crashOutput `json:"crash_output"` + Executable string `json:"executable"` + ExitCode int `json:"exit_code"` + LintTarget string `json:"lint_target"` + Stdstreams string `json:"stdstreams"` + Wd string `json:"wd"` + } + + meta := &metadata{ + Args: clangTidyCmd.Args, + CrashOutput: nil, + Executable: clangTidyCmd.Path, + ExitCode: exitCode, + LintTarget: cSrcFile, + Stdstreams: stdstreams.String(), + Wd: env.getwd(), + } + + // Sometimes, clang-tidy crashes. Unfortunately, these don't get funnelled through the + // standard clang crash machinery. :(. Try to work with our own. + if crashArtifactsDir != "" && strings.Contains(meta.Stdstreams, clangTidyCrashSubstring) { + tidyCrashArtifacts := path.Join(crashArtifactsDir, "clang-tidy") + if err := os.MkdirAll(tidyCrashArtifacts, 0777); err != nil { + return fmt.Errorf("creating crash artifacts directory at %q: %v", tidyCrashArtifacts, err) + } + + f, err := ioutil.TempFile(tidyCrashArtifacts, "crash-") + if err != nil { + return fmt.Errorf("making tempfile for crash output: %v", err) + } + f.Close() + + reproCmd := &command{} + *reproCmd = *clangCmd + reproCmd.Args = append(reproCmd.Args, "-E", "-o", f.Name()) + + reproOut := &strings.Builder{} + _, err = wrapSubprocessErrorWithSourceLoc(reproCmd, env.run(reproCmd, nil, reproOut, reproOut)) + if err != nil { + return fmt.Errorf("attempting to produce a clang-tidy crash reproducer: %v", err) + } + meta.CrashOutput = &crashOutput{ + CrashReproducerPath: f.Name(), + Stdstreams: reproOut.String(), + } + } + + f, err = os.Create(fixesMetadataPath) + if err != nil { + return fmt.Errorf("creating fixes metadata: %v", err) + } + + if err := json.NewEncoder(f).Encode(meta); err != nil { + return fmt.Errorf("writing fixes metadata: %v", err) + } + + if err := f.Close(); err != nil { + return fmt.Errorf("finalizing fixes metadata: %v", err) + } + return nil +} + +func runClangTidy(env env, clangCmd *command, cSrcFile string, extraTidyFlags []string) error { + extraTidyFlags = append(extraTidyFlags, + "-checks="+strings.Join([]string{ + "*", + "-bugprone-narrowing-conversions", + "-cppcoreguidelines-*", + "-fuchsia-*", + "-google-readability*", + "-google-runtime-references", + "-hicpp-*", + "-llvm-*", + "-misc-non-private-member-variables-in-classes", + "-misc-unused-parameters", + "-modernize-*", + "-readability-*", + }, ",")) + clangTidyCmd, err := calcClangTidyInvocation(env, clangCmd, cSrcFile, extraTidyFlags...) + if err != nil { + return fmt.Errorf("calculating clang-tidy invocation: %v", err) } // Note: We pass nil as stdin as we checked before that the compiler diff --git a/compiler_wrapper/clang_tidy_flag_test.go b/compiler_wrapper/clang_tidy_flag_test.go index baf5219e..4293bb21 100644 --- a/compiler_wrapper/clang_tidy_flag_test.go +++ b/compiler_wrapper/clang_tidy_flag_test.go @@ -273,6 +273,176 @@ func TestPartiallyOmitGomaWithClangTidy(t *testing.T) { }) } +func TestTriciumClangTidyIsProperlyDetectedFromEnv(t *testing.T) { + withClangTidyTestContext(t, func(ctx *testContext) { + ctx.env = []string{"WITH_TIDY=tricium"} + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } + return nil + case 2: + if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil { + return err + } + + hasFixesFile := false + for _, arg := range cmd.Args { + if path := strings.TrimPrefix(arg, "--export-fixes="); path != arg { + hasFixesFile = true + if !strings.HasPrefix(path, ctx.cfg.triciumNitsDir+"/") { + t.Errorf("fixes file was %q; expected it to be in %q", path, ctx.cfg.triciumNitsDir) + } + break + } + } + + if !hasFixesFile { + t.Error("no fixes file was provided to a tricium invocation") + } + + return nil + default: + return nil + } + } + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, mainCc))) + if ctx.cmdCount != 3 { + t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount) + } + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } + }) +} + +func TestTriciumClangTidySkipsProtobufFiles(t *testing.T) { + withClangTidyTestContext(t, func(ctx *testContext) { + ctx.env = []string{"WITH_TIDY=tricium"} + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, mainCc+".pb.cc"))) + if ctx.cmdCount != 1 { + t.Errorf("expected tricium clang-tidy to not execute on a protobuf file") + } + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } + }) +} + +func testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t *testing.T, ctx *testContext) { + addedFlag := "--some_clang_tidy=flag" + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } else if err := verifyArgCount(cmd, 0, addedFlag); err != nil { + t.Error(err) + } + return nil + case 2: + if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil { + t.Error(err) + } else if verifyArgCount(cmd, 1, addedFlag); err != nil { + t.Error(err) + } + return nil + default: + return nil + } + } + cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc, "-clang-tidy-flag="+addedFlag))) + if ctx.cmdCount != 3 { + t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount) + } + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } +} + +func TestClangTidyFiltersClangTidySpecificFlagsForTricium(t *testing.T) { + withClangTidyTestContext(t, func(ctx *testContext) { + ctx.env = []string{"WITH_TIDY=tricium"} + testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t, ctx) + }) +} + +func TestClangTidyFiltersClangTidySpecificFlags(t *testing.T) { + withClangTidyTestContext(t, func(ctx *testContext) { + testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t, ctx) + }) +} + +func TestClangTidyFlagsAreFilteredFromGccInvocations(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc, "-clang-tidy-flag=--foo"))) + if err := verifyArgCount(cmd, 0, ".*--foo.*"); err != nil { + t.Error(err) + } + }) +} + +func TestTriciumReportsClangTidyCrashesGracefully(t *testing.T) { + withClangTidyTestContext(t, func(ctx *testContext) { + ctx.env = []string{"WITH_TIDY=tricium"} + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } + return nil + case 2: + if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil { + return err + } + + if _, err := io.WriteString(stdout, clangTidyCrashSubstring); err != nil { + return err + } + return nil + case 3: + if err := verifyPath(cmd, "usr/bin/clang"); err != nil { + t.Error(err) + } + + args := cmd.Args + if len(args) < 3 { + t.Errorf("insufficient number of args provided; got %d; want at least 3", len(args)) + return nil + } + + lastArgs := args[len(args)-3:] + eArg, oArg, outFileArg := lastArgs[0], lastArgs[1], lastArgs[2] + if eArg != "-E" { + t.Errorf("got eArg=%q; wanted -E", eArg) + } + + if oArg != "-o" { + t.Errorf("got oArg=%q; wanted -o", oArg) + } + + wantPrefix := path.Join(ctx.cfg.crashArtifactsDir, "clang-tidy") + if !strings.HasPrefix(outFileArg, wantPrefix) { + t.Errorf("got out file %q; wanted one starting with %q", outFileArg, wantPrefix) + } + + return nil + default: + return nil + } + } + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc))) + if ctx.cmdCount != 4 { + t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount) + } + }) +} + func withClangTidyTestContext(t *testing.T, work func(ctx *testContext)) { withTestContext(t, func(ctx *testContext) { ctx.env = []string{"WITH_TIDY=1"} diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go index 4c3db14a..2b9459e4 100644 --- a/compiler_wrapper/compiler_wrapper.go +++ b/compiler_wrapper/compiler_wrapper.go @@ -80,6 +80,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int case clangType: mainBuilder.addPreUserArgs(mainBuilder.cfg.clangFlags...) mainBuilder.addPreUserArgs(mainBuilder.cfg.commonFlags...) + mainBuilder.addPostUserArgs(mainBuilder.cfg.clangPostFlags...) if _, err := processGomaCccFlags(mainBuilder); err != nil { return 0, err } @@ -89,40 +90,55 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int default: return 0, newErrorwithSourceLocf("unsupported compiler: %s", mainBuilder.target.compiler) } - } else if mainBuilder.target.compilerType == clangType { - cSrcFile, useClangTidy := processClangTidyFlags(mainBuilder) - err := prepareClangCommand(mainBuilder) - if err != nil { - return 0, err - } - allowCCache := true - if useClangTidy { - allowCCache = false - clangCmdWithoutGomaAndCCache := mainBuilder.build() - if err := runClangTidy(env, clangCmdWithoutGomaAndCCache, cSrcFile); err != nil { - return 0, err - } - } - if err := processGomaCCacheFlags(allowCCache, mainBuilder); err != nil { - return 0, err - } - compilerCmd = mainBuilder.build() } else { - if clangSyntax { - allowCCache := false - clangCmd, err := calcClangCommand(allowCCache, mainBuilder.clone()) + cSrcFile, tidyFlags, tidyMode := processClangTidyFlags(mainBuilder) + if mainBuilder.target.compilerType == clangType { + err := prepareClangCommand(mainBuilder) if err != nil { return 0, err } - gccCmd, err := calcGccCommand(mainBuilder) + allowCCache := true + if tidyMode != tidyModeNone { + allowCCache = false + clangCmdWithoutGomaAndCCache := mainBuilder.build() + var err error + switch tidyMode { + case tidyModeTricium: + if cfg.triciumNitsDir == "" { + return 0, newErrorwithSourceLocf("tricium linting was requested, but no nits directory is configured") + } + err = runClangTidyForTricium(env, clangCmdWithoutGomaAndCCache, cSrcFile, cfg.triciumNitsDir, tidyFlags, cfg.crashArtifactsDir) + case tidyModeAll: + err = runClangTidy(env, clangCmdWithoutGomaAndCCache, cSrcFile, tidyFlags) + default: + panic(fmt.Sprintf("Unknown tidy mode: %v", tidyMode)) + } + + if err != nil { + return 0, err + } + } + if err := processGomaCCacheFlags(allowCCache, mainBuilder); err != nil { + return 0, err + } + compilerCmd = mainBuilder.build() + } else { + if clangSyntax { + allowCCache := false + clangCmd, err := calcClangCommand(allowCCache, mainBuilder.clone()) + if err != nil { + return 0, err + } + gccCmd, err := calcGccCommand(mainBuilder) + if err != nil { + return 0, err + } + return checkClangSyntax(env, clangCmd, gccCmd) + } + compilerCmd, err = calcGccCommand(mainBuilder) if err != nil { return 0, err } - return checkClangSyntax(env, clangCmd, gccCmd) - } - compilerCmd, err = calcGccCommand(mainBuilder) - if err != nil { - return 0, err } } rusageLogfileName := getRusageLogFilename(env) @@ -167,6 +183,9 @@ func prepareClangCommand(builder *commandBuilder) (err error) { processSysrootFlag(builder) } builder.addPreUserArgs(builder.cfg.clangFlags...) + if builder.cfg.crashArtifactsDir != "" { + builder.addPreUserArgs("-fcrash-diagnostics-dir=" + builder.cfg.crashArtifactsDir) + } builder.addPostUserArgs(builder.cfg.clangPostFlags...) calcCommonPreUserArgs(builder) return processClangFlags(builder) diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go index a122b5a2..194ab579 100644 --- a/compiler_wrapper/config.go +++ b/compiler_wrapper/config.go @@ -29,6 +29,10 @@ type config struct { rootRelPath string // Directory to store errors that were prevented with -Wno-error. newWarningsDir string + // Directory to store nits in when using `WITH_TIDY=tricium`. + triciumNitsDir string + // Directory to store crash artifacts in. + crashArtifactsDir string // Version. Only used for printing via -print-cmd. version string } @@ -92,6 +96,7 @@ func getConfig(configName string, useCCache bool, useLlvmNext bool, version stri cfg.useLlvmNext = useLlvmNext if useLlvmNext { cfg.clangFlags = append(cfg.clangFlags, llvmNextFlags...) + cfg.clangPostFlags = append(cfg.clangPostFlags, llvmNextPostFlags...) } cfg.version = version return &cfg, nil @@ -119,9 +124,10 @@ var crosHardenedConfig = &config{ // Disable "-faddrsig" since it produces object files that strip doesn't understand, chromium:915742. // Pass "-fcommon" till the packages are fixed to work with new clang default // "-fno-common", crbug.com/1060413. + // crbug.com/1103065: -grecord-gcc-switches pollutes the Goma cache; + // removed that flag for now. clangFlags: []string{ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -132,11 +138,14 @@ var crosHardenedConfig = &config{ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", }, clangPostFlags: []string{ "-Wno-implicit-int-float-conversion", }, - newWarningsDir: "/tmp/fatal_clang_warnings", + newWarningsDir: "/tmp/fatal_clang_warnings", + triciumNitsDir: "/tmp/linting_output/clang-tidy", + crashArtifactsDir: "/tmp/clang_crash_diagnostics", } // Flags to be added to non-hardened toolchain. @@ -161,11 +170,14 @@ var crosNonHardenedConfig = &config{ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", }, clangPostFlags: []string{ "-Wno-implicit-int-float-conversion", }, - newWarningsDir: "/tmp/fatal_clang_warnings", + newWarningsDir: "/tmp/fatal_clang_warnings", + triciumNitsDir: "/tmp/linting_output/clang-tidy", + crashArtifactsDir: "/tmp/clang_crash_diagnostics", } // Flags to be added to host toolchain. @@ -182,9 +194,10 @@ var crosHostConfig = &config{ // Temporarily add no-unknown-warning-option to deal with old clang versions. // Pass "-fcommon" till the packages are fixed to work with new clang default // "-fno-common", crbug.com/1060413. + // crbug.com/1103065: -grecord-gcc-switches pollutes the Goma cache; + // removed that flag for now. clangFlags: []string{ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -195,20 +208,25 @@ var crosHostConfig = &config{ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", }, clangPostFlags: []string{ "-Wno-implicit-int-float-conversion", }, - newWarningsDir: "/tmp/fatal_clang_warnings", + newWarningsDir: "/tmp/fatal_clang_warnings", + triciumNitsDir: "/tmp/linting_output/clang-tidy", + crashArtifactsDir: "/tmp/clang_crash_diagnostics", } var androidConfig = &config{ - isHostWrapper: false, - isAndroidWrapper: true, - rootRelPath: "./", - commonFlags: []string{}, - gccFlags: []string{}, - clangFlags: []string{}, - clangPostFlags: []string{}, - newWarningsDir: "", + isHostWrapper: false, + isAndroidWrapper: true, + rootRelPath: "./", + commonFlags: []string{}, + gccFlags: []string{}, + clangFlags: []string{}, + clangPostFlags: []string{}, + newWarningsDir: "", + triciumNitsDir: "", + crashArtifactsDir: "", } diff --git a/compiler_wrapper/cros_llvm_next_flags.go b/compiler_wrapper/cros_llvm_next_flags.go index 6cd7cd2a..f5823fbf 100644 --- a/compiler_wrapper/cros_llvm_next_flags.go +++ b/compiler_wrapper/cros_llvm_next_flags.go @@ -13,6 +13,11 @@ package main // tag is set. // TODO: Enable test in config_test.go, once we have new llvm-next flags. -var llvmNextFlags = []string{} +var llvmNextFlags = []string{ + "-Wno-compound-token-split-by-macro", +} -var llvmNextPostFlags = []string{} +var llvmNextPostFlags = []string{ + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", +} diff --git a/compiler_wrapper/update_compiler_wrapper.sh b/compiler_wrapper/install_compiler_wrapper.sh index 479b112c..479b112c 100755 --- a/compiler_wrapper/update_compiler_wrapper.sh +++ b/compiler_wrapper/install_compiler_wrapper.sh diff --git a/compiler_wrapper/libc_exec.go b/compiler_wrapper/libc_exec.go index f8db9d86..d9867733 100644 --- a/compiler_wrapper/libc_exec.go +++ b/compiler_wrapper/libc_exec.go @@ -7,15 +7,31 @@ package main // #include <errno.h> +// #include <stdio.h> // #include <stdlib.h> // #include <string.h> // #include <unistd.h> +// #include <sys/types.h> +// #include <sys/wait.h> // // int libc_exec(const char *pathname, char *const argv[], char *const envp[]) { -// if (execve(pathname, argv, envp) != 0) { +// // Since fork() brings us to one thread, we can only use async-signal-safe funcs below. +// pid_t pid = fork(); +// if (pid == 0) { +// execve(pathname, argv, envp); +// fprintf(stderr, "exec failed (errno: %d)\n", errno); +// _exit(1); +// } +// if (pid < 0) { +// return errno; +// } +// +// int wstatus; +// pid_t waited = waitpid(pid, &wstatus, 0); +// if (waited == -1) { // return errno; // } -// return 0; +// exit(WEXITSTATUS(wstatus)); //} import "C" import ( @@ -29,7 +45,7 @@ import ( // Note that this changes the go binary to be a dynamically linked one. // See crbug.com/1000863 for details. // To use this version of exec, please add '-tags libc_exec' when building Go binary. -// Without the tags, libc_exec.go will be used. +// Without the tags, libc_exec.go will not be used. func execCmd(env env, cmd *command) error { freeList := []unsafe.Pointer{} diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json index 606fecda..cc55bc4f 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json @@ -25,7 +25,6 @@ "/tmp/sysroot_bisect", "/tmp/stable/clang", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -36,6 +35,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ], @@ -73,7 +74,6 @@ "someBisectDir", "/tmp/stable/clang", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -84,6 +84,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ], @@ -124,7 +126,6 @@ "someBisectDir", "/tmp/stable/clang", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -135,6 +136,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ], diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json index 7f45584a..5a59277c 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json @@ -16,7 +16,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -27,6 +26,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -51,7 +52,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -62,6 +62,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -86,7 +88,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -97,6 +98,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -121,7 +124,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -132,6 +134,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -156,7 +160,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -167,6 +170,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -191,7 +196,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -202,6 +206,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -226,7 +232,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -237,6 +242,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -261,7 +268,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -272,6 +278,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -296,7 +304,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -307,6 +314,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json index afb0329e..44b2fb91 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json @@ -15,7 +15,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -26,6 +25,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json index 05b0ff96..0b951f3e 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json @@ -15,7 +15,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -26,6 +25,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -49,7 +50,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -60,6 +60,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -83,7 +85,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -94,6 +95,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -117,7 +120,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -128,6 +130,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -151,7 +155,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -162,6 +165,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -185,7 +190,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -196,6 +200,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -219,7 +225,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -230,6 +235,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -253,7 +260,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -264,6 +270,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -287,7 +295,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -298,6 +305,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json index f7924a06..69fb4410 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json @@ -15,7 +15,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -26,6 +25,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -52,7 +53,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -63,6 +63,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -89,7 +91,6 @@ "path": "/tmp/stable/clang++", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -100,6 +101,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -126,7 +129,6 @@ "path": "somepath/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -137,6 +139,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -170,7 +174,6 @@ "path": "/somedir/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -181,6 +184,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", @@ -219,7 +224,6 @@ "path": "/somedir/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -230,6 +234,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", @@ -268,7 +274,6 @@ "path": "/somedir/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -279,6 +284,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", @@ -307,7 +314,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -318,6 +324,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -341,7 +349,6 @@ "path": "/tmp/stable/a/b/c/d/e/f/g/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -352,6 +359,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -375,7 +384,6 @@ "path": "/tmp/stable/a/b/c/d/e/f/g/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -386,6 +394,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -409,7 +419,6 @@ "path": "/tmp/stable/somedir/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -420,6 +429,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -446,7 +457,6 @@ "path": "/tmp/stable/pathenv/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -457,6 +467,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json index 11ebc821..93e566fe 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json @@ -17,7 +17,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -28,6 +27,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion" @@ -54,7 +55,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -65,6 +65,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion" @@ -91,7 +93,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -102,6 +103,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion" @@ -128,7 +131,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -139,6 +141,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion" @@ -164,7 +168,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -175,6 +178,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fno-experimental-new-pass-manager", "-fsanitize=fuzzer", "main.cc", @@ -202,7 +207,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -213,6 +217,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fno-experimental-new-pass-manager", "-fsanitize=address", "-fprofile-instr-generate", @@ -240,7 +246,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -251,6 +256,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=address", "main.cc", "-Wno-implicit-int-float-conversion" @@ -276,7 +283,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -287,6 +293,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fprofile-instr-generate", "main.cc", "-Wno-implicit-int-float-conversion" diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json index 7a724c88..41143bf0 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json @@ -26,7 +26,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -37,6 +36,16 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", + "-mno-movbe", + "-Wclobbered", + "-Wno-psabi", + "-Wlogical-op", + "-Wmissing-parameter-type", + "-Wold-style-declaration", + "-Woverride-init", + "-Wunsafe-loop-optimizations", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -61,7 +70,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -72,6 +80,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-#warnings", "main.cc", "-Wno-implicit-int-float-conversion" @@ -97,7 +107,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -108,6 +117,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-error=uninitialized", "main.cc", "-Wno-implicit-int-float-conversion" @@ -133,7 +144,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -144,6 +154,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-error=unused-variable", "main.cc", "-Wno-implicit-int-float-conversion" @@ -169,7 +181,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -180,6 +191,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-unused-variable", "main.cc", "-Wno-implicit-int-float-conversion" @@ -205,7 +218,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -216,6 +228,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wunused-variable", "main.cc", "-Wno-implicit-int-float-conversion" @@ -241,7 +255,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -252,6 +265,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-someflag", "main.cc", "-Wno-implicit-int-float-conversion" diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json index b4cd44c2..c4f86b01 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json @@ -31,7 +31,6 @@ "--", "-resource-dir=someResourceDir", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -42,6 +41,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -52,7 +53,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -63,6 +63,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -103,7 +105,6 @@ "--", "-resource-dir=someResourceDir", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -114,6 +115,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -124,7 +127,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -135,6 +137,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -177,7 +181,6 @@ "--", "-resource-dir=someResourceDir", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -188,6 +191,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -201,7 +206,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -212,6 +216,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -255,7 +261,6 @@ "--", "-resource-dir=someResourceDir", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -266,6 +271,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -276,7 +283,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -287,6 +293,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json index 66e58dc9..272b4220 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json @@ -18,7 +18,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -29,6 +28,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -55,7 +56,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -66,6 +66,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -78,7 +80,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -89,6 +90,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", "-Wno-error", @@ -119,7 +122,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -130,6 +132,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion" ] @@ -142,7 +146,6 @@ "path": "/tmp/stable/clang", "args": [ "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-fuse-ld=lld", @@ -153,6 +156,8 @@ "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", "-Wno-unknown-warning-option", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", "-Wno-error", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index 5b95cdb7..97dec849 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -27,7 +27,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -38,6 +37,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -45,6 +46,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -88,7 +90,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -99,6 +100,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -106,6 +109,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -152,7 +156,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -163,6 +166,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -170,6 +175,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json index aa083d70..6223ed91 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json @@ -18,7 +18,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -29,6 +28,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -37,6 +38,7 @@ "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -69,7 +71,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-eabi", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -80,6 +81,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -88,6 +91,7 @@ "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-eabi" @@ -120,7 +124,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-win-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -131,6 +134,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -139,6 +144,7 @@ "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-win-gnu" @@ -171,7 +177,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -182,6 +187,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -222,7 +229,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-eabi", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -233,6 +239,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -273,7 +281,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-win-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -284,6 +291,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -324,7 +333,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -335,6 +343,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -375,7 +385,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-eabi", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -386,6 +395,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -426,7 +437,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-win-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -437,6 +447,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json index 257c0a05..717681c4 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json @@ -17,7 +17,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -28,6 +27,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -35,6 +36,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -66,7 +68,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-eabi", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -77,6 +78,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -84,6 +87,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-eabi" @@ -115,7 +119,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-win-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -126,6 +129,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -133,6 +138,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-win-gnu" @@ -164,7 +170,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -175,6 +180,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -213,7 +220,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-eabi", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -224,6 +230,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -262,7 +270,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-win-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -273,6 +280,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -311,7 +320,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -322,6 +330,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -360,7 +370,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-eabi", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -371,6 +380,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -409,7 +420,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-win-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -420,6 +430,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json index b9edc49d..6940fcaf 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json @@ -17,7 +17,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -28,6 +27,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -35,6 +36,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -69,7 +71,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -80,6 +81,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -87,6 +90,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -121,7 +125,6 @@ "../../usr/bin/clang++", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -132,6 +135,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -139,6 +144,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -173,7 +179,6 @@ "somepath/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -184,6 +189,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -191,6 +198,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -232,7 +240,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -243,6 +250,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -252,6 +261,7 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -296,7 +306,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -307,6 +316,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -316,6 +327,7 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -355,7 +367,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -366,6 +377,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -375,6 +388,7 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -409,7 +423,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -420,6 +433,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -427,6 +442,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -458,7 +474,6 @@ "a/b/usr/bin/clang", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -469,6 +484,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -476,6 +493,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -507,7 +525,6 @@ "a/b/usr/bin/clang", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -518,6 +535,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -525,6 +544,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -556,7 +576,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -567,6 +586,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -574,6 +595,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -608,7 +630,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -619,6 +640,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -626,6 +649,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json index 2d70dfbf..acf6c24c 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json @@ -19,7 +19,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -30,6 +29,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -37,6 +38,7 @@ "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -70,7 +72,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -81,6 +82,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -88,6 +91,7 @@ "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -121,7 +125,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -132,6 +135,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -139,6 +144,7 @@ "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -172,7 +178,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -183,6 +188,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -190,6 +197,7 @@ "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -222,7 +230,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -233,6 +240,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -241,6 +250,7 @@ "-fsanitize=fuzzer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -274,7 +284,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -285,6 +294,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -294,6 +305,7 @@ "-fprofile-instr-generate", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -326,7 +338,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -337,6 +348,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -344,6 +357,7 @@ "-fsanitize=address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -376,7 +390,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -387,6 +400,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -395,6 +410,7 @@ "-fprofile-instr-generate", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json index 50f71a6f..26f5a665 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json @@ -28,7 +28,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -39,13 +38,24 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", + "-mno-movbe", + "-Wclobbered", + "-Wno-psabi", + "-Wlogical-op", + "-Wmissing-parameter-type", + "-Wold-style-declaration", + "-Woverride-init", + "-Wunsafe-loop-optimizations", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -78,7 +88,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -89,6 +98,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -97,6 +108,7 @@ "-Wno-#warnings", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -129,7 +141,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -140,6 +151,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -148,6 +161,7 @@ "-Wno-error=uninitialized", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -180,7 +194,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -191,6 +204,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -199,6 +214,7 @@ "-Wno-error=unused-variable", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -231,7 +247,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -242,6 +257,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -250,6 +267,7 @@ "-Wno-unused-variable", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -282,7 +300,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -293,6 +310,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -301,6 +320,7 @@ "-Wunused-variable", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -333,7 +353,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -344,6 +363,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -352,6 +373,7 @@ "-someflag", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json index b690e3ed..247f207c 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json @@ -52,7 +52,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -63,6 +62,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -70,6 +71,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -104,7 +106,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -115,6 +116,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -122,6 +125,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -149,7 +153,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -160,11 +163,14 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -197,7 +203,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -208,12 +213,15 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-fno-stack-protector", "-D__KERNEL__", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -246,7 +254,6 @@ "../../usr/bin/clang", "--sysroot=/usr/armv7a-cros-linux-gnueabihf", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -257,6 +264,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-D_FORTIFY_SOURCE=2", "-mthumb", "-fno-stack-protector", @@ -294,7 +303,6 @@ "args": [ "../../usr/bin/clang", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -305,6 +313,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -313,6 +323,7 @@ "--sysroot=xyz", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json index 26abab87..219a8b2f 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json @@ -32,7 +32,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -43,6 +42,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -50,6 +51,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -62,7 +64,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -73,6 +74,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -80,6 +83,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -122,7 +126,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -133,6 +136,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -140,6 +145,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -153,7 +159,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -164,6 +169,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -171,6 +178,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -215,7 +223,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -226,6 +233,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -233,6 +242,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -249,7 +259,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -260,6 +269,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -267,6 +278,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -312,7 +324,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -323,6 +334,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -330,6 +343,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -343,7 +357,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -354,6 +367,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -361,6 +376,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json index 61a63a37..10e485da 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json @@ -20,7 +20,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -31,6 +30,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -38,6 +39,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -72,7 +74,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -83,6 +84,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -90,6 +93,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -110,7 +114,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -121,6 +124,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -128,6 +133,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -166,7 +172,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -177,6 +182,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -184,6 +191,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -204,7 +212,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -215,6 +222,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -222,6 +231,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json index a37725c0..ea804d97 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json @@ -17,7 +17,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -28,6 +27,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -35,6 +36,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -90,7 +92,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -101,6 +102,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -108,6 +111,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -158,7 +162,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -169,6 +172,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -176,6 +181,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -210,7 +216,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -221,6 +226,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -228,6 +235,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index 5b95cdb7..b8e0e462 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -27,7 +27,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -38,6 +37,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -45,6 +47,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -88,7 +93,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -99,6 +103,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -106,6 +113,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -152,7 +162,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -163,6 +172,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -170,6 +182,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json index b9edc49d..9780356e 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json @@ -17,7 +17,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -28,6 +27,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -35,6 +37,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -69,7 +74,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -80,6 +84,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -87,6 +94,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -121,7 +131,6 @@ "../../usr/bin/clang++", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -132,6 +141,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -139,6 +151,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -173,7 +188,6 @@ "somepath/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -184,6 +198,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -191,6 +208,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -232,7 +252,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -243,6 +262,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -252,6 +274,9 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -296,7 +321,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -307,6 +331,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -316,6 +343,9 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -355,7 +385,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -366,6 +395,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -375,6 +407,9 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -409,7 +444,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -420,6 +454,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -427,6 +464,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -458,7 +498,6 @@ "a/b/usr/bin/clang", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -469,6 +508,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -476,6 +518,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -507,7 +552,6 @@ "a/b/usr/bin/clang", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -518,6 +562,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -525,6 +572,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -556,7 +606,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -567,6 +616,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -574,6 +626,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -608,7 +663,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -619,6 +673,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -626,6 +683,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json index 26abab87..06e114cb 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json @@ -32,7 +32,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -43,6 +42,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -50,6 +52,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -62,7 +67,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -73,6 +77,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -80,6 +87,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -122,7 +132,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -133,6 +142,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -140,6 +152,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -153,7 +168,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -164,6 +178,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -171,6 +188,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -215,7 +235,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -226,6 +245,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -233,6 +255,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -249,7 +274,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -260,6 +284,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -267,6 +294,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -312,7 +342,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -323,6 +352,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -330,6 +362,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -343,7 +378,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -354,6 +388,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -361,6 +398,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json index 61a63a37..cd460a79 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json @@ -20,7 +20,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -31,6 +30,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -38,6 +40,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -72,7 +77,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -83,6 +87,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -90,6 +97,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -110,7 +120,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -121,6 +130,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -128,6 +140,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -166,7 +181,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -177,6 +191,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -184,6 +201,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -204,7 +224,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -215,6 +234,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -222,6 +244,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json index a37725c0..c7f619d9 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json @@ -17,7 +17,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -28,6 +27,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -35,6 +37,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -90,7 +95,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -101,6 +105,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -108,6 +115,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -158,7 +168,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -169,6 +178,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -176,6 +188,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -210,7 +225,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -221,6 +235,9 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-Wno-compound-token-split-by-macro", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -228,6 +245,9 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-Wno-compound-token-split-by-space", + "-Wno-string-concatenation", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index e9bc3661..af4e3ff5 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -26,7 +26,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -37,6 +36,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -44,6 +45,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -83,7 +85,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -94,6 +95,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -101,6 +104,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -143,7 +147,6 @@ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -154,6 +157,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -161,6 +166,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json index c21d6198..50423288 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json @@ -16,7 +16,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -27,6 +26,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -34,6 +35,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -62,7 +64,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -73,6 +74,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -80,6 +83,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -108,7 +112,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -119,6 +122,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -126,6 +131,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -154,7 +160,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -165,6 +170,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -172,6 +179,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -207,7 +215,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -218,6 +225,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -227,6 +236,7 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -266,7 +276,6 @@ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -277,6 +286,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -286,6 +297,7 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -324,7 +336,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -335,6 +346,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -344,6 +357,7 @@ "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -372,7 +386,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -383,6 +396,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -390,6 +405,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -415,7 +431,6 @@ "args": [ "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -426,6 +441,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -433,6 +450,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -458,7 +476,6 @@ "args": [ "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -469,6 +486,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -476,6 +495,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -501,7 +521,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -512,6 +531,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -519,6 +540,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -547,7 +569,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -558,6 +579,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -565,6 +588,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json index 26abab87..219a8b2f 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json @@ -32,7 +32,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -43,6 +42,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -50,6 +51,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -62,7 +64,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -73,6 +74,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -80,6 +83,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -122,7 +126,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -133,6 +136,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -140,6 +145,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -153,7 +159,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -164,6 +169,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -171,6 +178,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -215,7 +223,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -226,6 +233,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -233,6 +242,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -249,7 +259,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -260,6 +269,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -267,6 +278,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -312,7 +324,6 @@ "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -323,6 +334,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -330,6 +343,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -343,7 +357,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -354,6 +367,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -361,6 +376,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json index ce35f311..a7fa8525 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json @@ -19,7 +19,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -30,6 +29,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -37,6 +38,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -65,7 +67,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -76,6 +77,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -83,6 +86,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -97,7 +101,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -108,6 +111,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -115,6 +120,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -147,7 +153,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -158,6 +163,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -165,6 +172,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -179,7 +187,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -190,6 +197,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -197,6 +206,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json index f716dde3..bafca635 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json @@ -17,7 +17,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -28,6 +27,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -35,6 +36,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -85,7 +87,6 @@ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -96,6 +97,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -103,6 +106,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -153,7 +157,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -164,6 +167,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -171,6 +176,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -205,7 +211,6 @@ "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", "-Qunused-arguments", - "-grecord-gcc-switches", "-fno-addrsig", "-fcommon", "-Wno-tautological-constant-compare", @@ -216,6 +221,8 @@ "-fuse-ld=lld", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fstack-protector-strong", "-fPIE", "-pie", @@ -223,6 +230,7 @@ "-fno-omit-frame-pointer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index ce83cd72..8b03c092 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -34,8 +34,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -86,8 +89,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -141,8 +147,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json index db5bea28..5d72756d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json @@ -25,9 +25,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -67,9 +70,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-eabi" @@ -109,9 +115,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-win-gnu" @@ -151,6 +160,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "-ftrapv", "main.cc", @@ -194,6 +205,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", @@ -236,6 +249,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "-ftrapv", "main.cc", @@ -279,6 +294,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "-ftrapv", "main.cc", @@ -322,6 +339,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-ftrapv", "main.cc", "-Wno-implicit-int-float-conversion", @@ -364,6 +383,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "-ftrapv", "main.cc", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json index 8b5d7088..382b7097 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json @@ -24,8 +24,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -64,8 +67,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-eabi" @@ -104,8 +110,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-win-gnu" @@ -144,6 +153,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "main.cc", "-Wno-implicit-int-float-conversion", @@ -185,6 +196,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", "-B../../bin", @@ -225,6 +238,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "main.cc", "-Wno-implicit-int-float-conversion", @@ -266,6 +281,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "main.cc", "-Wno-implicit-int-float-conversion", @@ -307,6 +324,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", "-B../../bin", @@ -347,6 +366,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "main.cc", "-Wno-implicit-int-float-conversion", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json index 418f9d62..cadfd1fe 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json @@ -24,8 +24,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -67,8 +70,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -110,8 +116,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -153,8 +162,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -203,10 +215,13 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -258,10 +273,13 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -308,10 +326,13 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -353,8 +374,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -393,8 +417,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -433,8 +460,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-Ba/b/bin", "-target", "x86_64-cros-linux-gnu" @@ -473,8 +503,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -516,8 +549,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json index 54a3268c..8786db27 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json @@ -26,9 +26,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -69,9 +72,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -112,9 +118,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -155,9 +164,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=kernel-address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -197,10 +209,13 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fno-experimental-new-pass-manager", "-fsanitize=fuzzer", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -241,11 +256,14 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fno-experimental-new-pass-manager", "-fsanitize=address", "-fprofile-instr-generate", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -285,9 +303,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fsanitize=address", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -327,9 +348,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fprofile-instr-generate", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json index 4a30d74e..98ca5a67 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json @@ -35,8 +35,19 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", + "-mno-movbe", + "-Wclobbered", + "-Wno-psabi", + "-Wlogical-op", + "-Wmissing-parameter-type", + "-Wold-style-declaration", + "-Woverride-init", + "-Wunsafe-loop-optimizations", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -76,9 +87,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-#warnings", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -118,9 +132,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-error=uninitialized", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -160,9 +177,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-error=unused-variable", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -202,9 +222,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wno-unused-variable", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -244,9 +267,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-Wunused-variable", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -286,9 +312,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-someflag", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json index 8b5c8b90..d39e2b0a 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json @@ -55,8 +55,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -98,8 +101,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -134,8 +140,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -175,10 +184,13 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fno-stack-protector", "-D__KERNEL__", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -218,6 +230,8 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-mthumb", "-fno-stack-protector", "-D__KERNEL__", @@ -261,9 +275,12 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "--sysroot=xyz", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json index cf30514c..ce9659a2 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json @@ -39,8 +39,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -60,8 +63,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -111,8 +117,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -133,8 +142,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -186,8 +198,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -211,8 +226,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -265,8 +283,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -287,8 +308,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json index 7aad322f..61bff7d7 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json @@ -27,8 +27,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -70,8 +73,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -99,8 +105,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -146,8 +155,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu" @@ -175,8 +187,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json index 76d81fa9..244e8fad 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json @@ -24,8 +24,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -84,8 +87,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -139,8 +145,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", @@ -182,8 +191,11 @@ "-static-libgcc", "-Wno-final-dtor-non-final-class", "-Werror=poison-system-directories", + "-fexperimental-new-pass-manager", + "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "main.cc", "-Wno-implicit-int-float-conversion", + "-mno-movbe", "-B../../bin", "-target", "x86_64-cros-linux-gnu", diff --git a/compiler_wrapper/testutil_test.go b/compiler_wrapper/testutil_test.go index 5d8ef920..21b7169d 100644 --- a/compiler_wrapper/testutil_test.go +++ b/compiler_wrapper/testutil_test.go @@ -141,6 +141,8 @@ func (ctx *testContext) mustFail(exitCode int) string { func (ctx *testContext) updateConfig(cfg *config) { *ctx.cfg = *cfg ctx.cfg.newWarningsDir = filepath.Join(ctx.tempDir, "fatal_clang_warnings") + ctx.cfg.triciumNitsDir = filepath.Join(ctx.tempDir, "tricium_nits") + ctx.cfg.crashArtifactsDir = filepath.Join(ctx.tempDir, "clang_crash_diagnostics") } func (ctx *testContext) newCommand(path string, args ...string) *command { diff --git a/cros_utils/buildbot_json.py b/cros_utils/buildbot_json.py deleted file mode 100755 index 08a8ae05..00000000 --- a/cros_utils/buildbot_json.py +++ /dev/null @@ -1,1534 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2019 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# NOTE: This file is NOT under GPL. See above. - -"""Queries buildbot through the json interface. -""" - -from __future__ import print_function - -__author__ = 'maruel@chromium.org' -__version__ = '1.2' - -import code -import datetime -import functools -import json - -# Pylint recommends we use "from chromite.lib import cros_logging as logging". -# Chromite specific policy message, we want to keep using the standard logging. -# pylint: disable=cros-logging-import -import logging - -# pylint: disable=deprecated-module -import optparse - -import time -import sys -import urllib.error -import urllib.parse -import urllib.request - -try: - from natsort import natsorted -except ImportError: - # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted - # after "vm7". Defaults to normal sorting. - natsorted = sorted - -# These values are buildbot constants used for Build and BuildStep. -# This line was copied from master/buildbot/status/builder.py. -SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = list(range(6)) - -## Generic node caching code. - - -class Node(object): - """Root class for all nodes in the graph. - - Provides base functionality for any node in the graph, independent if it has - children or not or if its content can be addressed through an url or needs to - be fetched as part of another node. - - self.printable_attributes is only used for self documentation and for str() - implementation. - """ - printable_attributes = [] - - def __init__(self, parent, url): - self.printable_attributes = self.printable_attributes[:] - if url: - self.printable_attributes.append('url') - url = url.rstrip('/') - if parent is not None: - self.printable_attributes.append('parent') - self.url = url - self.parent = parent - - def __str__(self): - return self.to_string() - - def __repr__(self): - """Embeds key if present.""" - key = getattr(self, 'key', None) - if key is not None: - return '<%s key=%s>' % (self.__class__.__name__, key) - cached_keys = getattr(self, 'cached_keys', None) - if cached_keys is not None: - return '<%s keys=%s>' % (self.__class__.__name__, cached_keys) - return super(Node, self).__repr__() - - def to_string(self, maximum=100): - out = ['%s:' % self.__class__.__name__] - assert not 'printable_attributes' in self.printable_attributes - - def limit(txt): - txt = str(txt) - if maximum > 0: - if len(txt) > maximum + 2: - txt = txt[:maximum] + '...' - return txt - - for k in sorted(self.printable_attributes): - if k == 'parent': - # Avoid infinite recursion. - continue - out.append(limit(' %s: %r' % (k, getattr(self, k)))) - return '\n'.join(out) - - def refresh(self): - """Refreshes the data.""" - self.discard() - return self.cache() - - def cache(self): # pragma: no cover - """Caches the data.""" - raise NotImplementedError() - - def discard(self): # pragma: no cover - """Discards cached data. - - Pretty much everything is temporary except completed Build. - """ - raise NotImplementedError() - - -class AddressableBaseDataNode(Node): # pylint: disable=W0223 - """A node that contains a dictionary of data that can be fetched with an url. - - The node is directly addressable. It also often can be fetched by the parent. - """ - printable_attributes = Node.printable_attributes + ['data'] - - def __init__(self, parent, url, data): - super(AddressableBaseDataNode, self).__init__(parent, url) - self._data = data - - @property - def cached_data(self): - return self._data - - @property - def data(self): - self.cache() - return self._data - - def cache(self): - if self._data is None: - self._data = self._readall() - return True - return False - - def discard(self): - self._data = None - - def read(self, suburl): - assert self.url, self.__class__.__name__ - url = self.url - if suburl: - url = '%s/%s' % (self.url, suburl) - return self.parent.read(url) - - def _readall(self): - return self.read('') - - -class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223 - """Automatically encodes the url.""" - - def __init__(self, parent, url, data): - super(AddressableDataNode, self).__init__(parent, urllib.parse.quote(url), - data) - - -class NonAddressableDataNode(Node): # pylint: disable=W0223 - """A node that cannot be addressed by an unique url. - - The data comes directly from the parent. - """ - - def __init__(self, parent, subkey): - super(NonAddressableDataNode, self).__init__(parent, None) - self.subkey = subkey - - @property - def cached_data(self): - if self.parent.cached_data is None: - return None - return self.parent.cached_data[self.subkey] - - @property - def data(self): - return self.parent.data[self.subkey] - - def cache(self): - self.parent.cache() - - def discard(self): # pragma: no cover - """Avoid invalid state when parent recreate the object.""" - raise AttributeError('Call parent discard() instead') - - -class VirtualNodeList(Node): - """Base class for every node that has children. - - Adds partial supports for keys and iterator functionality. 'key' can be a - string or a int. Not to be used directly. - """ - printable_attributes = Node.printable_attributes + ['keys'] - - def __init__(self, parent, url): - super(VirtualNodeList, self).__init__(parent, url) - # Keeps the keys independently when ordering is needed. - self._is_cached = False - self._has_keys_cached = False - - def __contains__(self, key): - """Enables 'if i in obj:'.""" - return key in self.keys - - def __iter__(self): - """Enables 'for i in obj:'. It returns children.""" - self.cache_keys() - for key in self.keys: - yield self[key] - - def __len__(self): - """Enables 'len(obj)' to get the number of childs.""" - return len(self.keys) - - def discard(self): - """Discards data. - - The default behavior is to not invalidate cached keys. The only place where - keys need to be invalidated is with Builds. - """ - self._is_cached = False - self._has_keys_cached = False - - @property - def cached_children(self): # pragma: no cover - """Returns an iterator over the children that are cached.""" - raise NotImplementedError() - - @property - def cached_keys(self): # pragma: no cover - raise NotImplementedError() - - @property - def keys(self): # pragma: no cover - """Returns the keys for every children.""" - raise NotImplementedError() - - def __getitem__(self, key): # pragma: no cover - """Returns a child, without fetching its data. - - The children could be invalid since no verification is done. - """ - raise NotImplementedError() - - def cache(self): # pragma: no cover - """Cache all the children.""" - raise NotImplementedError() - - def cache_keys(self): # pragma: no cover - """Cache all children's keys.""" - raise NotImplementedError() - - -class NodeList(VirtualNodeList): # pylint: disable=W0223 - """Adds a cache of the keys.""" - - def __init__(self, parent, url): - super(NodeList, self).__init__(parent, url) - self._keys = [] - - @property - def cached_keys(self): - return self._keys - - @property - def keys(self): - self.cache_keys() - return self._keys - - -class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223 - """A node that contains children but retrieves all its data from its parent. - - I.e. there's no url to get directly this data. - """ - # Child class object for children of this instance. For example, BuildSteps - # has BuildStep children. - _child_cls = None - - def __init__(self, parent, subkey): - super(NonAddressableNodeList, self).__init__(parent, None) - self.subkey = subkey - assert (not isinstance(self._child_cls, NonAddressableDataNode) and - issubclass(self._child_cls, NonAddressableDataNode)), ( - self._child_cls.__name__) - - @property - def cached_children(self): - if self.parent.cached_data is not None: - for i in range(len(self.parent.cached_data[self.subkey])): - yield self[i] - - @property - def cached_data(self): - if self.parent.cached_data is None: - return None - return self.parent.data.get(self.subkey, None) - - @property - def cached_keys(self): - if self.parent.cached_data is None: - return None - return list(range(len(self.parent.data.get(self.subkey, [])))) - - @property - def data(self): - return self.parent.data[self.subkey] - - def cache(self): - self.parent.cache() - - def cache_keys(self): - self.parent.cache() - - def discard(self): # pragma: no cover - """Do not call. - - Avoid infinite recursion by having the caller calls the parent's - discard() explicitely. - """ - raise AttributeError('Call parent discard() instead') - - def __iter__(self): - """Enables 'for i in obj:'. It returns children.""" - if self.data: - for i in range(len(self.data)): - yield self[i] - - def __getitem__(self, key): - """Doesn't cache the value, it's not needed. - - TODO(maruel): Cache? - """ - if isinstance(key, int) and key < 0: - key = len(self.data) + key - # pylint: disable=E1102 - return self._child_cls(self, key) - - -class AddressableNodeList(NodeList): - """A node that has children that can be addressed with an url.""" - - # Child class object for children of this instance. For example, Builders has - # Builder children and Builds has Build children. - _child_cls = None - - def __init__(self, parent, url): - super(AddressableNodeList, self).__init__(parent, url) - self._cache = {} - assert (not isinstance(self._child_cls, AddressableDataNode) and - issubclass(self._child_cls, AddressableDataNode)), ( - self._child_cls.__name__) - - @property - def cached_children(self): - for item in self._cache.values(): - if item.cached_data is not None: - yield item - - @property - def cached_keys(self): - return list(self._cache.keys()) - - def __getitem__(self, key): - """Enables 'obj[i]'.""" - if self._has_keys_cached and not key in self._keys: - raise KeyError(key) - - if not key in self._cache: - # Create an empty object. - self._create_obj(key, None) - return self._cache[key] - - def cache(self): - if not self._is_cached: - data = self._readall() - for key in sorted(data): - self._create_obj(key, data[key]) - self._is_cached = True - self._has_keys_cached = True - - def cache_partial(self, children): - """Caches a partial number of children. - - This method is more efficient since it does a single request for all the - children instead of one request per children. - - It only grab objects not already cached. - """ - # pylint: disable=W0212 - if not self._is_cached: - to_fetch = [ - child for child in children - if not (child in self._cache and self._cache[child].cached_data) - ] - if to_fetch: - # Similar to cache(). The only reason to sort is to simplify testing. - params = '&'.join( - 'select=%s' % urllib.parse.quote(str(v)) for v in sorted(to_fetch)) - data = self.read('?' + params) - for key in sorted(data): - self._create_obj(key, data[key]) - - def cache_keys(self): - """Implement to speed up enumeration. Defaults to call cache().""" - if not self._has_keys_cached: - self.cache() - assert self._has_keys_cached - - def discard(self): - """Discards temporary children.""" - super(AddressableNodeList, self).discard() - for v in self._cache.values(): - v.discard() - - def read(self, suburl): - assert self.url, self.__class__.__name__ - url = self.url - if suburl: - url = '%s/%s' % (self.url, suburl) - return self.parent.read(url) - - def _create_obj(self, key, data): - """Creates an object of type self._child_cls.""" - # pylint: disable=E1102 - obj = self._child_cls(self, key, data) - # obj.key and key may be different. - # No need to overide cached data with None. - if data is not None or obj.key not in self._cache: - self._cache[obj.key] = obj - if obj.key not in self._keys: - self._keys.append(obj.key) - - def _readall(self): - return self.read('') - - -class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223 - """A node that shows a subset of children that comes from another structure. - - The node is not addressable. - - E.g. the keys are retrieved from parent but the actual data comes from - virtual_parent. - """ - - def __init__(self, parent, virtual_parent, subkey): - super(SubViewNodeList, self).__init__(parent, None) - self.subkey = subkey - self.virtual_parent = virtual_parent - assert isinstance(self.parent, AddressableDataNode) - assert isinstance(self.virtual_parent, NodeList) - - @property - def cached_children(self): - if self.parent.cached_data is not None: - for item in self.keys: - if item in self.virtual_parent.keys: - child = self[item] - if child.cached_data is not None: - yield child - - @property - def cached_keys(self): - return (self.parent.cached_data or {}).get(self.subkey, []) - - @property - def keys(self): - self.cache_keys() - return self.parent.data.get(self.subkey, []) - - def cache(self): - """Batch request for each child in a single read request.""" - if not self._is_cached: - self.virtual_parent.cache_partial(self.keys) - self._is_cached = True - - def cache_keys(self): - if not self._has_keys_cached: - self.parent.cache() - self._has_keys_cached = True - - def discard(self): - if self.parent.cached_data is not None: - for child in self.virtual_parent.cached_children: - if child.key in self.keys: - child.discard() - self.parent.discard() - super(SubViewNodeList, self).discard() - - def __getitem__(self, key): - """Makes sure the key is in our key but grab it from the virtual parent.""" - return self.virtual_parent[key] - - def __iter__(self): - self.cache() - return super(SubViewNodeList, self).__iter__() - - -# Buildbot-specific code - - -class Slave(AddressableDataNode): - """Buildbot slave class.""" - printable_attributes = AddressableDataNode.printable_attributes + [ - 'name', - 'key', - 'connected', - 'version', - ] - - def __init__(self, parent, name, data): - super(Slave, self).__init__(parent, name, data) - self.name = name - self.key = self.name - # TODO(maruel): Add SlaveBuilders and a 'builders' property. - # TODO(maruel): Add a 'running_builds' property. - - @property - def connected(self): - return self.data.get('connected', False) - - @property - def version(self): - return self.data.get('version') - - -class Slaves(AddressableNodeList): - """Buildbot slaves.""" - _child_cls = Slave - printable_attributes = AddressableNodeList.printable_attributes + ['names'] - - def __init__(self, parent): - super(Slaves, self).__init__(parent, 'slaves') - - @property - def names(self): - return self.keys - - -class BuilderSlaves(SubViewNodeList): - """Similar to Slaves but only list slaves connected to a specific builder.""" - printable_attributes = SubViewNodeList.printable_attributes + ['names'] - - def __init__(self, parent): - super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves, - 'slaves') - - @property - def names(self): - return self.keys - - -class BuildStep(NonAddressableDataNode): - """Class for a buildbot build step.""" - printable_attributes = NonAddressableDataNode.printable_attributes + [ - 'name', - 'number', - 'start_time', - 'end_time', - 'duration', - 'is_started', - 'is_finished', - 'is_running', - 'result', - 'simplified_result', - ] - - def __init__(self, parent, number): - """Pre-loaded, since the data is retrieved via the Build object.""" - assert isinstance(number, int) - super(BuildStep, self).__init__(parent, number) - self.number = number - - @property - def start_time(self): - if self.data.get('times'): - return int(round(self.data['times'][0])) - - @property - def end_time(self): - times = self.data.get('times') - if times and len(times) == 2 and times[1]: - return int(round(times[1])) - - @property - def duration(self): - if self.start_time: - return (self.end_time or int(round(time.time()))) - self.start_time - - @property - def name(self): - return self.data['name'] - - @property - def is_started(self): - return self.data.get('isStarted', False) - - @property - def is_finished(self): - return self.data.get('isFinished', False) - - @property - def is_running(self): - return self.is_started and not self.is_finished - - @property - def result(self): - result = self.data.get('results') - if result is None: - # results may be 0, in that case with filter=1, the value won't be - # present. - if self.data.get('isFinished'): - result = self.data.get('results', 0) - while isinstance(result, list): - result = result[0] - return result - - @property - def simplified_result(self): - """Returns a simplified 3 state value, True, False or None.""" - result = self.result - if result in (SUCCESS, WARNINGS): - return True - elif result in (FAILURE, EXCEPTION, RETRY): - return False - assert result in (None, SKIPPED), (result, self.data) - return None - - -class BuildSteps(NonAddressableNodeList): - """Duplicates keys to support lookup by both step number and step name.""" - printable_attributes = NonAddressableNodeList.printable_attributes + [ - 'failed', - ] - _child_cls = BuildStep - - def __init__(self, parent): - """Pre-loaded, since the data is retrieved via the Build object.""" - super(BuildSteps, self).__init__(parent, 'steps') - - @property - def keys(self): - """Returns the steps name in order.""" - return [i['name'] for i in self.data or []] - - @property - def failed(self): - """Shortcuts that lists the step names of steps that failed.""" - return [step.name for step in self if step.simplified_result is False] - - def __getitem__(self, key): - """Accept step name in addition to index number.""" - if isinstance(key, str): - # It's a string, try to find the corresponding index. - for i, step in enumerate(self.data): - if step['name'] == key: - key = i - break - else: - raise KeyError(key) - return super(BuildSteps, self).__getitem__(key) - - -class Build(AddressableDataNode): - """Buildbot build info.""" - printable_attributes = AddressableDataNode.printable_attributes + [ - 'key', - 'number', - 'steps', - 'blame', - 'reason', - 'revision', - 'result', - 'simplified_result', - 'start_time', - 'end_time', - 'duration', - 'slave', - 'properties', - 'completed', - ] - - def __init__(self, parent, key, data): - super(Build, self).__init__(parent, str(key), data) - self.number = int(key) - self.key = self.number - self.steps = BuildSteps(self) - - @property - def blame(self): - return self.data.get('blame', []) - - @property - def builder(self): - """Returns the Builder object. - - Goes up the hierarchy to find the Buildbot.builders[builder] instance. - """ - return self.parent.parent.parent.parent.builders[self.data['builderName']] - - @property - def start_time(self): - if self.data.get('times'): - return int(round(self.data['times'][0])) - - @property - def end_time(self): - times = self.data.get('times') - if times and len(times) == 2 and times[1]: - return int(round(times[1])) - - @property - def duration(self): - if self.start_time: - return (self.end_time or int(round(time.time()))) - self.start_time - - @property - def eta(self): - return self.data.get('eta', 0) - - @property - def completed(self): - return self.data.get('currentStep') is None - - @property - def properties(self): - return self.data.get('properties', []) - - @property - def reason(self): - return self.data.get('reason') - - @property - def result(self): - result = self.data.get('results') - while isinstance(result, list): - result = result[0] - if result is None and self.steps: - # results may be 0, in that case with filter=1, the value won't be - # present. - result = self.steps[-1].result - return result - - @property - def revision(self): - return self.data.get('sourceStamp', {}).get('revision') - - @property - def simplified_result(self): - """Returns a simplified 3 state value, True, False or None.""" - result = self.result - if result in (SUCCESS, WARNINGS, SKIPPED): - return True - elif result in (FAILURE, EXCEPTION, RETRY): - return False - assert result is None, (result, self.data) - return None - - @property - def slave(self): - """Returns the Slave object. - - Goes up the hierarchy to find the Buildbot.slaves[slave] instance. - """ - return self.parent.parent.parent.parent.slaves[self.data['slave']] - - def discard(self): - """Completed Build isn't discarded.""" - if self._data and self.result is None: - assert not self.steps or not self.steps[-1].data.get('isFinished') - self._data = None - - -class CurrentBuilds(SubViewNodeList): - """Lists of the current builds.""" - - def __init__(self, parent): - super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds') - - -class PendingBuilds(AddressableDataNode): - """List of the pending builds.""" - - def __init__(self, parent): - super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None) - - -class Builds(AddressableNodeList): - """Supports iteration. - - Recommends using .cache() to speed up if a significant number of builds are - iterated over. - """ - _child_cls = Build - - def __init__(self, parent): - super(Builds, self).__init__(parent, 'builds') - - def __getitem__(self, key): - """Support for negative reference and enable retrieving non-cached builds. - - e.g. -1 is the last build, -2 is the previous build before the last one. - """ - key = int(key) - if key < 0: - # Convert negative to positive build number. - self.cache_keys() - # Since the negative value can be outside of the cache keys range, use the - # highest key value and calculate from it. - key = max(self._keys) + key + 1 - - if key not in self._cache: - # Create an empty object. - self._create_obj(key, None) - return self._cache[key] - - def __iter__(self): - """Returns cached Build objects in reversed order. - - The most recent build is returned first and then in reverse chronological - order, up to the oldest cached build by the server. Older builds can be - accessed but will trigger significantly more I/O so they are not included by - default in the iteration. - - To access the older builds, use self.iterall() instead. - """ - self.cache() - return reversed(list(self._cache.values())) - - def iterall(self): - """Returns Build objects in decreasing order unbounded up to build 0. - - The most recent build is returned first and then in reverse chronological - order. Older builds can be accessed and will trigger significantly more I/O - so use this carefully. - """ - # Only cache keys here. - self.cache_keys() - if self._keys: - for i in range(max(self._keys), -1, -1): - yield self[i] - - def cache_keys(self): - """Grabs the keys (build numbers) from the builder.""" - if not self._has_keys_cached: - for i in self.parent.data.get('cachedBuilds', []): - i = int(i) - self._cache.setdefault(i, Build(self, i, None)) - if i not in self._keys: - self._keys.append(i) - self._has_keys_cached = True - - def discard(self): - super(Builds, self).discard() - # Can't keep keys. - self._has_keys_cached = False - - def _readall(self): - return self.read('_all') - - -class Builder(AddressableDataNode): - """Builder status.""" - printable_attributes = AddressableDataNode.printable_attributes + [ - 'name', - 'key', - 'builds', - 'slaves', - 'pending_builds', - 'current_builds', - ] - - def __init__(self, parent, name, data): - super(Builder, self).__init__(parent, name, data) - self.name = name - self.key = name - self.builds = Builds(self) - self.slaves = BuilderSlaves(self) - self.current_builds = CurrentBuilds(self) - self.pending_builds = PendingBuilds(self) - - def discard(self): - super(Builder, self).discard() - self.builds.discard() - self.slaves.discard() - self.current_builds.discard() - - -class Builders(AddressableNodeList): - """Root list of builders.""" - _child_cls = Builder - - def __init__(self, parent): - super(Builders, self).__init__(parent, 'builders') - - -class Buildbot(AddressableBaseDataNode): - """This object should be recreated on a master restart as it caches data.""" - # Throttle fetches to not kill the server. - auto_throttle = None - printable_attributes = AddressableDataNode.printable_attributes + [ - 'slaves', - 'builders', - 'last_fetch', - ] - - def __init__(self, url): - super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None) - self._builders = Builders(self) - self._slaves = Slaves(self) - self.last_fetch = None - - @property - def builders(self): - return self._builders - - @property - def slaves(self): - return self._slaves - - def discard(self): - """Discards information about Builders and Slaves.""" - super(Buildbot, self).discard() - self._builders.discard() - self._slaves.discard() - - def read(self, suburl): - if self.auto_throttle: - if self.last_fetch: - delta = datetime.datetime.utcnow() - self.last_fetch - remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta) - if remaining > datetime.timedelta(seconds=0): - logging.debug('Sleeping for %ss', remaining) - time.sleep(remaining.seconds) - self.last_fetch = datetime.datetime.utcnow() - url = '%s/%s' % (self.url, suburl) - if '?' in url: - url += '&filter=1' - else: - url += '?filter=1' - logging.info('read(%s)', suburl) - channel = urllib.request.urlopen(url) - data = channel.read() - try: - return json.loads(data) - except ValueError: - if channel.getcode() >= 400: - # Convert it into an HTTPError for easier processing. - raise urllib.error.HTTPError(url, channel.getcode(), - '%s:\n%s' % (url, data), channel.headers, - None) - raise - - def _readall(self): - return self.read('project') - - -# Controller code - - -def usage(more): - - def hook(fn): - fn.func_usage_more = more - return fn - - return hook - - -def need_buildbot(fn): - """Post-parse args to create a buildbot object.""" - - @functools.wraps(fn) - def hook(parser, args, *extra_args, **kwargs): - old_parse_args = parser.parse_args - - def new_parse_args(args): - options, args = old_parse_args(args) - if len(args) < 1: - parser.error('Need to pass the root url of the buildbot') - url = args.pop(0) - if not url.startswith('http'): - url = 'http://' + url - buildbot = Buildbot(url) - buildbot.auto_throttle = options.throttle - return options, args, buildbot - - parser.parse_args = new_parse_args - # Call the original function with the modified parser. - return fn(parser, args, *extra_args, **kwargs) - - hook.func_usage_more = '[options] <url>' - return hook - - -@need_buildbot -def CMDpending(parser, args): - """Lists pending jobs.""" - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.builders: - options.builders = buildbot.builders.keys - for builder in options.builders: - builder = buildbot.builders[builder] - pending_builds = builder.data.get('pendingBuilds', 0) - if not pending_builds: - continue - print('Builder %s: %d' % (builder.name, pending_builds)) - if not options.quiet: - for pending in builder.pending_builds.data: - if 'revision' in pending['source']: - print(' revision: %s' % pending['source']['revision']) - for change in pending['source']['changes']: - print(' change:') - print(' comment: %r' % change['comments'][:50]) - print(' who: %s' % change['who']) - return 0 - - -@usage('[options] <url> [commands] ...') -@need_buildbot -def CMDrun(parser, args): - """Runs commands passed as parameters. - - When passing commands on the command line, each command will be run as if it - was on its own line. - """ - parser.add_option('-f', '--file', help='Read script from file') - parser.add_option( - '-i', dest='use_stdin', action='store_true', help='Read script on stdin') - # Variable 'buildbot' is not used directly. - # pylint: disable=W0612 - options, args, _ = parser.parse_args(args) - if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1: - parser.error('Need to pass only one of: <commands>, -f <file> or -i') - if options.use_stdin: - cmds = sys.stdin.read() - elif options.file: - cmds = open(options.file).read() - else: - cmds = '\n'.join(args) - compiled = compile(cmds, '<cmd line>', 'exec') - # pylint: disable=eval-used - eval(compiled, globals(), locals()) - return 0 - - -@need_buildbot -def CMDinteractive(parser, args): - """Runs an interactive shell to run queries.""" - _, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - prompt = ('Buildbot interactive console for "%s".\n' - "Hint: Start with typing: 'buildbot.printable_attributes' or " - "'print str(buildbot)' to explore.") % buildbot.url[:-len('/json')] - local_vars = {'buildbot': buildbot, 'b': buildbot} - code.interact(prompt, None, local_vars) - - -@need_buildbot -def CMDidle(parser, args): - """Lists idle slaves.""" - return find_idle_busy_slaves(parser, args, True) - - -@need_buildbot -def CMDbusy(parser, args): - """Lists idle slaves.""" - return find_idle_busy_slaves(parser, args, False) - - -@need_buildbot -def CMDdisconnected(parser, args): - """Lists disconnected slaves.""" - _, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - for slave in buildbot.slaves: - if not slave.connected: - print(slave.name) - return 0 - - -def find_idle_busy_slaves(parser, args, show_idle): - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '-s', - '--slave', - dest='slaves', - action='append', - default=[], - help='Slaves to filter on') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.builders: - options.builders = buildbot.builders.keys - for builder in options.builders: - builder = buildbot.builders[builder] - if options.slaves: - # Only the subset of slaves connected to the builder. - slaves = list(set(options.slaves).intersection(set(builder.slaves.names))) - if not slaves: - continue - else: - slaves = builder.slaves.names - busy_slaves = [build.slave.name for build in builder.current_builds] - if show_idle: - slaves = natsorted(set(slaves) - set(busy_slaves)) - else: - slaves = natsorted(set(slaves) & set(busy_slaves)) - if options.quiet: - for slave in slaves: - print(slave) - else: - if slaves: - print('Builder %s: %s' % (builder.name, ', '.join(slaves))) - return 0 - - -def last_failure(buildbot, - builders=None, - slaves=None, - steps=None, - no_cache=False): - """Returns Build object with last failure with the specific filters.""" - builders = builders or buildbot.builders.keys - for builder in builders: - builder = buildbot.builders[builder] - if slaves: - # Only the subset of slaves connected to the builder. - builder_slaves = list(set(slaves).intersection(set(builder.slaves.names))) - if not builder_slaves: - continue - else: - builder_slaves = builder.slaves.names - - if not no_cache and len(builder.slaves) > 2: - # Unless you just want the last few builds, it's often faster to - # fetch the whole thing at once, at the cost of a small hickup on - # the buildbot. - # TODO(maruel): Cache only N last builds or all builds since - # datetime. - builder.builds.cache() - - found = [] - for build in builder.builds: - if build.slave.name not in builder_slaves or build.slave.name in found: - continue - # Only add the slave for the first completed build but still look for - # incomplete builds. - if build.completed: - found.append(build.slave.name) - - if steps: - if any(build.steps[step].simplified_result is False for step in steps): - yield build - elif build.simplified_result is False: - yield build - - if len(found) == len(builder_slaves): - # Found all the slaves, quit. - break - - -@need_buildbot -def CMDlast_failure(parser, args): - """Lists all slaves that failed on that step on their last build. - - Examples: - To find all slaves where their last build was a compile failure, - run with --step compile - """ - parser.add_option( - '-S', - '--step', - dest='steps', - action='append', - default=[], - help='List all slaves that failed on that step on their last build') - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '-s', - '--slave', - dest='slaves', - action='append', - default=[], - help='Slaves to filter on') - parser.add_option( - '-n', - '--no_cache', - action='store_true', - help="Don't load all builds at once") - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - print_builders = not options.quiet and len(options.builders) != 1 - last_builder = None - for build in last_failure( - buildbot, - builders=options.builders, - slaves=options.slaves, - steps=options.steps, - no_cache=options.no_cache): - - if print_builders and last_builder != build.builder: - print(build.builder.name) - last_builder = build.builder - - if options.quiet: - if options.slaves: - print('%s: %s' % (build.builder.name, build.slave.name)) - else: - print(build.slave.name) - else: - out = '%d on %s: blame:%s' % (build.number, build.slave.name, ', '.join( - build.blame)) - if print_builders: - out = ' ' + out - print(out) - - if len(options.steps) != 1: - for step in build.steps: - if step.simplified_result is False: - # Assume the first line is the text name anyway. - summary = ', '.join(step.data['text'][1:])[:40] - out = ' %s: "%s"' % (step.data['name'], summary) - if print_builders: - out = ' ' + out - print(out) - return 0 - - -@need_buildbot -def CMDcurrent(parser, args): - """Lists current jobs.""" - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '--blame', action='store_true', help='Only print the blame list') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.builders: - options.builders = buildbot.builders.keys - - if options.blame: - blame = set() - for builder in options.builders: - for build in buildbot.builders[builder].current_builds: - if build.blame: - for blamed in build.blame: - blame.add(blamed) - print('\n'.join(blame)) - return 0 - - for builder in options.builders: - builder = buildbot.builders[builder] - if not options.quiet and builder.current_builds: - print(builder.name) - for build in builder.current_builds: - if options.quiet: - print(build.slave.name) - else: - out = '%4d: slave=%10s' % (build.number, build.slave.name) - out += ' duration=%5d' % (build.duration or 0) - if build.eta: - out += ' eta=%5.0f' % build.eta - else: - out += ' ' - if build.blame: - out += ' blame=' + ', '.join(build.blame) - print(out) - - return 0 - - -@need_buildbot -def CMDbuilds(parser, args): - """Lists all builds. - - Examples: - To find all builds on a single slave, run with -b bar -s foo. - """ - parser.add_option( - '-r', '--result', type='int', help='Build result to filter on') - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '-s', - '--slave', - dest='slaves', - action='append', - default=[], - help='Slaves to filter on') - parser.add_option( - '-n', - '--no_cache', - action='store_true', - help="Don't load all builds at once") - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - builders = options.builders or buildbot.builders.keys - for builder in builders: - builder = buildbot.builders[builder] - for build in builder.builds: - if not options.slaves or build.slave.name in options.slaves: - if options.quiet: - out = '' - if options.builders: - out += '%s/' % builder.name - if len(options.slaves) != 1: - out += '%s/' % build.slave.name - out += '%d revision:%s result:%s blame:%s' % ( - build.number, build.revision, build.result, ','.join(build.blame)) - print(out) - else: - print(build) - return 0 - - -@need_buildbot -def CMDcount(parser, args): - """Count the number of builds that occured during a specific period.""" - parser.add_option( - '-o', '--over', type='int', help='Number of seconds to look for') - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.over: - parser.error( - 'Specify the number of seconds, e.g. --over 86400 for the last 24 ' - 'hours') - builders = options.builders or buildbot.builders.keys - counts = {} - since = time.time() - options.over - for builder in builders: - builder = buildbot.builders[builder] - counts[builder.name] = 0 - if not options.quiet: - print(builder.name) - for build in builder.builds.iterall(): - try: - start_time = build.start_time - except urllib.error.HTTPError: - # The build was probably trimmed. - print( - 'Failed to fetch build %s/%d' % (builder.name, build.number), - file=sys.stderr) - continue - if start_time >= since: - counts[builder.name] += 1 - else: - break - if not options.quiet: - print('.. %d' % counts[builder.name]) - - align_name = max(len(b) for b in counts) - align_number = max(len(str(c)) for c in counts.values()) - for builder in sorted(counts): - print('%*s: %*d' % (align_name, builder, align_number, counts[builder])) - print('Total: %d' % sum(counts.values())) - return 0 - - -def gen_parser(): - """Returns an OptionParser instance with default options. - - It should be then processed with gen_usage() before being used. - """ - parser = optparse.OptionParser(version=__version__) - # Remove description formatting - parser.format_description = lambda x: parser.description - # Add common parsing. - old_parser_args = parser.parse_args - - def Parse(*args, **kwargs): - options, args = old_parser_args(*args, **kwargs) - if options.verbose >= 2: - logging.basicConfig(level=logging.DEBUG) - elif options.verbose: - logging.basicConfig(level=logging.INFO) - else: - logging.basicConfig(level=logging.WARNING) - return options, args - - parser.parse_args = Parse - - parser.add_option( - '-v', - '--verbose', - action='count', - help='Use multiple times to increase logging leve') - parser.add_option( - '-q', - '--quiet', - action='store_true', - help='Reduces the output to be parsed by scripts, independent of -v') - parser.add_option( - '--throttle', - type='float', - help='Minimum delay to sleep between requests') - return parser - - -# Generic subcommand handling code - - -def Command(name): - return getattr(sys.modules[__name__], 'CMD' + name, None) - - -@usage('<command>') -def CMDhelp(parser, args): - """Print list of commands or use 'help <command>'.""" - _, args = parser.parse_args(args) - if len(args) == 1: - return main(args + ['--help']) - parser.print_help() - return 0 - - -def gen_usage(parser, command): - """Modifies an OptionParser object with the command's documentation. - - The documentation is taken from the function's docstring. - """ - obj = Command(command) - more = getattr(obj, 'func_usage_more') - # OptParser.description prefer nicely non-formatted strings. - parser.description = obj.__doc__ + '\n' - parser.set_usage('usage: %%prog %s %s' % (command, more)) - - -def main(args=None): - # Do it late so all commands are listed. - # pylint: disable=E1101 - CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join( - ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0]) - for fn in dir(sys.modules[__name__]) - if fn.startswith('CMD')) - - parser = gen_parser() - if args is None: - args = sys.argv[1:] - if args: - command = Command(args[0]) - if command: - # "fix" the usage and the description now that we know the subcommand. - gen_usage(parser, args[0]) - return command(parser, args[1:]) - - # Not a known command. Default to help. - gen_usage(parser, 'help') - return CMDhelp(parser, args) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py index e0c54782..b600c6aa 100644 --- a/cros_utils/buildbot_utils.py +++ b/cros_utils/buildbot_utils.py @@ -50,8 +50,8 @@ def PeekTrybotImage(chromeos_root, buildbucket_id): and url looks like: gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 """ - command = ( - 'cros buildresult --report json --buildbucket-id %s' % buildbucket_id) + command = ('cros buildresult --report json --buildbucket-id %s' % + buildbucket_id) rc, out, _ = RunCommandInPath(chromeos_root, command) # Current implementation of cros buildresult returns fail when a job is still @@ -194,9 +194,9 @@ def GetTrybotImage(chromeos_root, image = '' if not image: - logger.GetLogger().LogError( - 'Trybot job (buildbucket id: %s) failed with' - 'status %s; no trybot image generated. ' % (buildbucket_id, status)) + logger.GetLogger().LogError('Trybot job (buildbucket id: %s) failed with' + 'status %s; no trybot image generated. ' % + (buildbucket_id, status)) else: # Convert full gs path to what crosperf expects. For example, convert # gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 @@ -226,13 +226,13 @@ def WaitForImage(chromeos_root, build): while elapsed_time < TIME_OUT: if DoesImageExist(chromeos_root, build): return - logger.GetLogger().LogOutput( - 'Image %s not ready, waiting for 10 minutes' % build) + logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' % + build) time.sleep(SLEEP_TIME) elapsed_time += SLEEP_TIME - logger.GetLogger().LogOutput( - 'Image %s not found, waited for %d hours' % (build, (TIME_OUT / 3600))) + logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' % + (build, (TIME_OUT / 3600))) raise BuildbotTimeout('Timeout while waiting for image %s' % build) @@ -253,7 +253,7 @@ def GetLatestImage(chromeos_root, path): candidates.sort(reverse=True) for c in candidates: build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3]) - # Blacklist "R79-12384.0.0" image released by mistake. + # Denylist "R79-12384.0.0" image released by mistake. # TODO(crbug.com/992242): Remove the filter by 2019-09-05. if c == [79, 12384, 0, 0]: continue diff --git a/cros_utils/buildbot_utils_unittest.py b/cros_utils/buildbot_utils_unittest.py index afbdfaef..c615c95f 100755 --- a/cros_utils/buildbot_utils_unittest.py +++ b/cros_utils/buildbot_utils_unittest.py @@ -21,18 +21,6 @@ from cros_utils import command_executer class TrybotTest(unittest.TestCase): """Test for CommandExecuter class.""" - old_tryjob_out = ( - 'Verifying patches...\n' - 'Submitting tryjob...\n' - 'Successfully sent PUT request to [buildbucket_bucket:master.chromiumos.t' - 'ryserver] with [config:success-build] [buildbucket_id:895272114382368817' - '6].\n' - 'Tryjob submitted!\n' - 'To view your tryjobs, visit:\n' - ' http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbuck' - 'etId=8952721143823688176\n' - ' https://uberchromegw.corp.google.com/i/chromiumos.tryserver/waterfall?' - 'committer=laszio@chromium.org&builder=etc\n') tryjob_out = ( '[{"buildbucket_id": "8952721143823688176", "build_config": ' '"cave-llvm-toolchain-tryjob", "url": ' diff --git a/cros_utils/command_executer.py b/cros_utils/command_executer.py index b9acbe31..aeedf3ea 100755 --- a/cros_utils/command_executer.py +++ b/cros_utils/command_executer.py @@ -229,9 +229,11 @@ class CommandExecuter(object): kwargs['return_output'] = True return self.RunCommandGeneric(*args, **kwargs) - def RemoteAccessInitCommand(self, chromeos_root, machine): + def RemoteAccessInitCommand(self, chromeos_root, machine, port=None): command = '' command += '\nset -- --remote=' + machine + if port: + command += ' --ssh_port=' + port command += '\n. ' + chromeos_root + '/src/scripts/common.sh' command += '\n. ' + chromeos_root + '/src/scripts/remote_access.sh' command += '\nTMP=$(mktemp -d)' @@ -285,12 +287,16 @@ class CommandExecuter(object): sys.exit(1) chromeos_root = os.path.expanduser(chromeos_root) + port = None + if ':' in machine: + machine, port = machine.split(':') # Write all commands to a file. command_file = self.WriteToTempShFile(cmd) retval = self.CopyFiles( command_file, command_file, dest_machine=machine, + dest_port=port, command_terminator=command_terminator, chromeos_root=chromeos_root, dest_cros=True, @@ -302,7 +308,7 @@ class CommandExecuter(object): ' Is the machine up?') return (retval, '', '') - command = self.RemoteAccessInitCommand(chromeos_root, machine) + command = self.RemoteAccessInitCommand(chromeos_root, machine, port) command += '\nremote_sh bash %s' % command_file command += '\nl_retval=$?; echo "$REMOTE_OUT"; exit $l_retval' retval = self.RunCommandGeneric( @@ -313,8 +319,8 @@ class CommandExecuter(object): terminated_timeout=terminated_timeout, print_to_console=print_to_console) if return_output: - connect_signature = ( - 'Initiating first contact with remote host\n' + 'Connection OK\n') + connect_signature = ('Initiating first contact with remote host\n' + + 'Connection OK\n') connect_signature_re = re.compile(connect_signature) modded_retval = list(retval) modded_retval[1] = connect_signature_re.sub('', retval[1]) @@ -381,7 +387,7 @@ class CommandExecuter(object): command_file = f.name os.chmod(command_file, 0o777) - # if return_output is set, run a dummy command first to make sure that + # if return_output is set, run a test command first to make sure that # the chroot already exists. We want the final returned output to skip # the output from chroot creation steps. if return_output: @@ -449,7 +455,9 @@ class CommandExecuter(object): src, dest, src_machine=None, + src_port=None, dest_machine=None, + dest_port=None, src_user=None, dest_user=None, recursive=True, @@ -475,30 +483,34 @@ class CommandExecuter(object): sys.exit(1) if src_cros: cros_machine = src_machine + cros_port = src_port + host_machine = dest_machine + host_user = dest_user else: cros_machine = dest_machine - - command = self.RemoteAccessInitCommand(chromeos_root, cros_machine) - ssh_command = ( - 'ssh -o StrictHostKeyChecking=no' + ' -o UserKnownHostsFile=$(mktemp)' - + ' -i $TMP_PRIVATE_KEY') + cros_port = dest_port + host_machine = src_machine + host_user = src_user + + command = self.RemoteAccessInitCommand(chromeos_root, cros_machine, + cros_port) + ssh_command = ('ssh -o StrictHostKeyChecking=no' + + ' -o UserKnownHostsFile=$(mktemp)' + + ' -i $TMP_PRIVATE_KEY') + if cros_port: + ssh_command += ' -p %s' % cros_port rsync_prefix = '\nrsync -r -e "%s" ' % ssh_command if dest_cros: - command += rsync_prefix + '%s root@%s:%s' % (src, dest_machine, dest) - return self.RunCommand( - command, - machine=src_machine, - username=src_user, - command_terminator=command_terminator, - print_to_console=print_to_console) + command += rsync_prefix + '%s root@%s:%s' % (src, cros_machine, dest) else: - command += rsync_prefix + 'root@%s:%s %s' % (src_machine, src, dest) - return self.RunCommand( - command, - machine=dest_machine, - username=dest_user, - command_terminator=command_terminator, - print_to_console=print_to_console) + command += rsync_prefix + 'root@%s:%s %s' % (cros_machine, src, dest) + + return self.RunCommand( + command, + machine=host_machine, + username=host_user, + command_terminator=command_terminator, + print_to_console=print_to_console) if dest_machine == src_machine: command = 'rsync -a %s %s' % (src, dest) diff --git a/cros_utils/manifest_versions.py b/cros_utils/manifest_versions.py index 115c6046..4838de3c 100644 --- a/cros_utils/manifest_versions.py +++ b/cros_utils/manifest_versions.py @@ -19,6 +19,8 @@ import time from cros_utils import command_executer from cros_utils import logger +MANIFEST_VERSION_MAIN_BRANCH = 'master' + def IsCrosVersion(version): match = re.search(r'(\d+\.\d+\.\d+\.\d+)', version) @@ -72,7 +74,8 @@ class ManifestVersions(object): commands = [ 'cd {0}'.format(self.clone_location), 'cd manifest-versions', 'git checkout -f $(git rev-list' + - ' --max-count=1 --before={0} origin/master)'.format(my_time) + ' --max-count=1 --before={0} origin/{1})'.format( + my_time, MANIFEST_VERSION_MAIN_BRANCH) ] ret = self.ce.RunCommands(commands) if ret: @@ -106,7 +109,7 @@ class ManifestVersions(object): version = pp[-2] + '.' + small commands = [ 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout master' + 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH) ] self.ce.RunCommands(commands) return version @@ -120,7 +123,8 @@ class ManifestVersions(object): commands = [ 'cd {0}'.format(self.clone_location), 'cd manifest-versions', 'git checkout -f $(git rev-list' + - ' --max-count=1 --before={0} origin/master)'.format(my_time) + ' --max-count=1 --before={0} origin/{1})'.format( + my_time, MANIFEST_VERSION_MAIN_BRANCH) ] ret = self.ce.RunCommands(commands) if ret: @@ -133,7 +137,7 @@ class ManifestVersions(object): version = pp[-2] + '.' + small commands = [ 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout master' + 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH) ] self.ce.RunCommands(commands) return version diff --git a/cros_utils/misc.py b/cros_utils/misc.py index b4beb60c..93d1b3b6 100644 --- a/cros_utils/misc.py +++ b/cros_utils/misc.py @@ -24,12 +24,15 @@ CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts' TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/' 'cros_utils/toolchain_utils.sh') +CROS_MAIN_BRANCH = 'cros/master' + def GetChromeOSVersionFromLSBVersion(lsb_version): """Get Chromeos version from Lsb version.""" ce = command_executer.GetCommandExecuter() command = ('git ls-remote ' - 'https://chromium.googlesource.com/chromiumos/manifest.git') + 'https://chromium.googlesource.com/chromiumos/manifest.git ' + 'refs/heads/release-R*') ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False) assert ret == 0, 'Command %s failed' % command lower = [] @@ -444,8 +447,9 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): cmd1, print_to_console=True) == 0 -def ApplyGerritPatches(chromeos_root, gerrit_patch_string, - branch='cros/master'): +def ApplyGerritPatches(chromeos_root, + gerrit_patch_string, + branch=CROS_MAIN_BRANCH): """Apply gerrit patches on a chromeos tree. Args: diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py index b4092993..1a3fd4a7 100644 --- a/cros_utils/tabulator.py +++ b/cros_utils/tabulator.py @@ -67,10 +67,10 @@ from __future__ import print_function import collections import getpass import math +import statistics import sys -# TODO(zhizhouy): Drop numpy in the future +# TODO(crbug.com/980719): Drop scipy in the future. # pylint: disable=import-error -import numpy import scipy from cros_utils.email_sender import EmailSender @@ -556,7 +556,7 @@ class AmeanResult(StringMeanResult): def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) - cell.value = numpy.mean(values) + cell.value = statistics.mean(values) class RawResult(Result): @@ -610,7 +610,7 @@ class StdResult(NumericalResult): def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) - cell.value = numpy.std(values) + cell.value = statistics.pstdev(values) class CoeffVarResult(NumericalResult): @@ -623,8 +623,8 @@ class CoeffVarResult(NumericalResult): def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) - if numpy.mean(values) != 0.0: - noise = numpy.abs(numpy.std(values) / numpy.mean(values)) + if statistics.mean(values) != 0.0: + noise = abs(statistics.pstdev(values) / statistics.mean(values)) else: noise = 0.0 cell.value = noise @@ -731,9 +731,12 @@ class AmeanRatioResult(KeyAwareComparisonResult): if self.ignore_min_max: values = _RemoveMinMax(cell, values) baseline_values = _RemoveMinMax(cell, baseline_values) - if numpy.mean(baseline_values) != 0: - cell.value = numpy.mean(values) / numpy.mean(baseline_values) - elif numpy.mean(values) != 0: + + baseline_mean = statistics.mean(baseline_values) + values_mean = statistics.mean(values) + if baseline_mean != 0: + cell.value = values_mean / baseline_mean + elif values_mean != 0: cell.value = 0.00 # cell.value = 0 means the values and baseline_values have big difference else: @@ -1495,41 +1498,40 @@ def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): if __name__ == '__main__': # Run a few small tests here. - runs = [ - [{ - 'k1': '10', - 'k2': '12', - 'k5': '40', - 'k6': '40', - 'ms_1': '20', - 'k7': 'FAIL', - 'k8': 'PASS', - 'k9': 'PASS', - 'k10': '0' - }, - { - 'k1': '13', - 'k2': '14', - 'k3': '15', - 'ms_1': '10', - 'k8': 'PASS', - 'k9': 'FAIL', - 'k10': '0' - }], - [{ - 'k1': '50', - 'k2': '51', - 'k3': '52', - 'k4': '53', - 'k5': '35', - 'k6': '45', - 'ms_1': '200', - 'ms_2': '20', - 'k7': 'FAIL', - 'k8': 'PASS', - 'k9': 'PASS' - }], - ] + run1 = { + 'k1': '10', + 'k2': '12', + 'k5': '40', + 'k6': '40', + 'ms_1': '20', + 'k7': 'FAIL', + 'k8': 'PASS', + 'k9': 'PASS', + 'k10': '0' + } + run2 = { + 'k1': '13', + 'k2': '14', + 'k3': '15', + 'ms_1': '10', + 'k8': 'PASS', + 'k9': 'FAIL', + 'k10': '0' + } + run3 = { + 'k1': '50', + 'k2': '51', + 'k3': '52', + 'k4': '53', + 'k5': '35', + 'k6': '45', + 'ms_1': '200', + 'ms_2': '20', + 'k7': 'FAIL', + 'k8': 'PASS', + 'k9': 'PASS' + } + runs = [[run1, run2], [run3]] labels = ['vanilla', 'modified'] t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) print(t) diff --git a/cros_utils/tabulator_test.py b/cros_utils/tabulator_test.py index 227e2d70..9dd4828e 100755 --- a/cros_utils/tabulator_test.py +++ b/cros_utils/tabulator_test.py @@ -33,6 +33,20 @@ class TabulatorTest(unittest.TestCase): result.Compute(cell, table[2], table[1]) self.assertTrue(cell.value == float(table[2][0])) + def testStdResult(self): + table = ['k1', [], ['1', '2']] + result = tabulator.StdResult() + cell = tabulator.Cell() + result.Compute(cell, table[2], table[1]) + self.assertTrue(cell.value == 0.5) + + def testStdResultOfSampleSize1(self): + table = ['k1', [], ['1']] + result = tabulator.StdResult() + cell = tabulator.Cell() + result.Compute(cell, table[2], table[1]) + self.assertTrue(cell.value == 0.0) + def testStringMean(self): smr = tabulator.StringMeanResult() cell = tabulator.Cell() diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 7b59c2af..cc7f33ef 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,8 +1,9 @@ -elm : chromeos2-row9-rack8-host19.cros chromeos2-row9-rack8-host21.cros -bob : chromeos2-row9-rack7-host1.cros chromeos2-row9-rack7-host3.cros -chell : chromeos2-row9-rack8-host3.cros chromeos2-row9-rack8-host5.cros -kefka : chromeos2-row9-rack9-host21.cros chromeos2-row9-rack8-host1.cros -lulu : chromeos2-row9-rack8-host9.cros chromeos2-row9-rack8-host7.cros -nautilus : chromeos2-row9-rack7-host11.cros chromeos2-row9-rack7-host9.cros -snappy : chromeos2-row9-rack7-host5.cros chromeos2-row9-rack7-host7.cros -veyron_minnie : chromeos2-row9-rack8-host15.cros chromeos2-row9-rack8-host17.cros +bob : chromeos2-row10-rack9-host1.cros chromeos2-row10-rack9-host3.cros +coral : chromeos2-row9-rack9-host9.cros chromeos2-row9-rack9-host11.cros chromeos2-row9-rack9-host13.cros +elm : chromeos2-row10-rack9-host19.cros chromeos2-row10-rack9-host21.cros +chell : chromeos2-row9-rack9-host1.cros chromeos2-row9-rack9-host3.cros +kefka : chromeos2-row9-rack9-host21.cros chromeos2-row10-rack9-host13.cros +lulu : chromeos2-row9-rack9-host5.cros chromeos2-row9-rack9-host7.cros +nautilus : chromeos2-row10-rack9-host9.cros chromeos2-row10-rack9-host11.cros +snappy : chromeos2-row10-rack9-host5.cros chromeos2-row10-rack9-host7.cros +veyron_minnie : chromeos2-row10-rack9-host15.cros chromeos2-row10-rack9-host17.cros diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 21fa3ea0..8ba85a4c 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -35,8 +35,8 @@ def _WriteJSONReportToFile(experiment, results_dir, json_report): compiler_string = 'llvm' if has_llvm else 'gcc' board = experiment.labels[0].board filename = 'report_%s_%s_%s.%s.json' % (board, json_report.date, - json_report.time.replace(':', '.'), - compiler_string) + json_report.time.replace( + ':', '.'), compiler_string) fullname = os.path.join(results_dir, filename) report_text = json_report.GetReport() with open(fullname, 'w') as out_file: @@ -157,13 +157,13 @@ class ExperimentRunner(object): def _ClearCacheEntries(self, experiment): for br in experiment.benchmark_runs: cache = ResultsCache() - cache.Init( - br.label.chromeos_image, br.label.chromeos_root, - br.benchmark.test_name, br.iteration, br.test_args, br.profiler_args, - br.machine_manager, br.machine, br.label.board, br.cache_conditions, - br.logger(), br.log_level, br.label, br.share_cache, - br.benchmark.suite, br.benchmark.show_all_results, - br.benchmark.run_local, br.benchmark.cwp_dso) + cache.Init(br.label.chromeos_image, br.label.chromeos_root, + br.benchmark.test_name, br.iteration, br.test_args, + br.profiler_args, br.machine_manager, br.machine, + br.label.board, br.cache_conditions, br.logger(), br.log_level, + br.label, br.share_cache, br.benchmark.suite, + br.benchmark.show_all_results, br.benchmark.run_local, + br.benchmark.cwp_dso) cache_dir = cache.GetCacheDirForWrite() if os.path.exists(cache_dir): self.l.LogOutput('Removing cache dir: %s' % cache_dir) @@ -173,7 +173,7 @@ class ExperimentRunner(object): try: # We should not lease machines if tests are launched via `skylab # create-test`. This is because leasing DUT in skylab will create a - # dummy task on the DUT and new test created will be hanging there. + # no-op task on the DUT and new test created will be hanging there. # TODO(zhizhouy): Need to check whether machine is ready or not before # assigning a test to it. if not experiment.skylab: @@ -246,8 +246,8 @@ class ExperimentRunner(object): subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names)) text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() - text_report += ( - '\nResults are stored in %s.\n' % experiment.results_directory) + text_report += ('\nResults are stored in %s.\n' % + experiment.results_directory) text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report html_report = HTMLResultsReport.FromExperiment(experiment).GetReport() attachment = EmailSender.Attachment('report.html', html_report) @@ -275,8 +275,8 @@ class ExperimentRunner(object): all_failed = True topstats_file = os.path.join(results_directory, 'topstats.log') - self.l.LogOutput( - 'Storing top statistics of each benchmark run into %s.' % topstats_file) + self.l.LogOutput('Storing top statistics of each benchmark run into %s.' % + topstats_file) with open(topstats_file, 'w') as top_fd: for benchmark_run in experiment.benchmark_runs: if benchmark_run.result: @@ -322,8 +322,8 @@ class ExperimentRunner(object): self.l.LogOutput('Storing email message body in %s.' % results_directory) msg_file_path = os.path.join(results_directory, 'msg_body.html') text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() - text_report += ( - '\nResults are stored in %s.\n' % experiment.results_directory) + text_report += ('\nResults are stored in %s.\n' % + experiment.results_directory) msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report FileUtils().WriteFile(msg_file_path, msg_body) @@ -348,8 +348,8 @@ class MockExperimentRunner(ExperimentRunner): super(MockExperimentRunner, self).__init__(experiment, json_report) def _Run(self, experiment): - self.l.LogOutput( - "Would run the following experiment: '%s'." % experiment.name) + self.l.LogOutput("Would run the following experiment: '%s'." % + experiment.name) def _PrintTable(self, experiment): self.l.LogOutput('Would print the experiment table.') diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py index e19d4695..8c3510a9 100755 --- a/crosperf/generate_report_unittest.py +++ b/crosperf/generate_report_unittest.py @@ -109,9 +109,9 @@ class GenerateReportTests(unittest.TestCase): } results = generate_report.CutResultsInPlace( bench_data, max_keys=0, complain_on_update=False) - # Just reach into results assuming we know it otherwise outputs things - # sanely. If it doesn't, testCutResultsInPlace should give an indication as - # to what, exactly, is broken. + # Just reach into results assuming we know it otherwise outputs things in + # the expected way. If it doesn't, testCutResultsInPlace should give an + # indication as to what, exactly, is broken. self.assertEqual(list(results['foo'][0][0].items()), [('retval', 0)]) self.assertEqual(list(results['bar'][0][0].items()), [('retval', 1)]) self.assertEqual(list(results['baz'][0][0].items()), []) diff --git a/crosperf/label.py b/crosperf/label.py index b8122613..a55d663c 100644 --- a/crosperf/label.py +++ b/crosperf/label.py @@ -61,9 +61,9 @@ class Label(object): if self.image_type == 'local': chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image) if not chromeos_root: - raise RuntimeError( - "No ChromeOS root given for label '%s' and could " - "not determine one from image path: '%s'." % (name, chromeos_image)) + raise RuntimeError("No ChromeOS root given for label '%s' and could " + "not determine one from image path: '%s'." % + (name, chromeos_image)) else: chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root) if not chromeos_root: @@ -72,17 +72,31 @@ class Label(object): self.chromeos_root = chromeos_root if not chrome_src: - self.chrome_src = os.path.join( - self.chromeos_root, '.cache/distfiles/target/chrome-src-internal') - if not os.path.exists(self.chrome_src): - self.chrome_src = os.path.join(self.chromeos_root, - '.cache/distfiles/target/chrome-src') + # Old and new chroots may have different chrome src locations. + # The path also depends on the chrome build flags. + # Give priority to chrome-src-internal. + chrome_src_rel_paths = [ + '.cache/distfiles/target/chrome-src-internal', + '.cache/distfiles/chrome-src-internal', + '.cache/distfiles/target/chrome-src', + '.cache/distfiles/chrome-src', + ] + for chrome_src_rel_path in chrome_src_rel_paths: + chrome_src_abs_path = os.path.join(self.chromeos_root, + chrome_src_rel_path) + if os.path.exists(chrome_src_abs_path): + chrome_src = chrome_src_abs_path + break + if not chrome_src: + raise RuntimeError('Can not find location of Chrome sources.\n' + f'Checked paths: {chrome_src_rel_paths}') else: - chromeos_src = misc.CanonicalizePath(chrome_src) - if not chromeos_src: + chrome_src = misc.CanonicalizePath(chrome_src) + # Make sure the path exists. + if not os.path.exists(chrome_src): raise RuntimeError("Invalid Chrome src given for label '%s': '%s'." % (name, chrome_src)) - self.chrome_src = chromeos_src + self.chrome_src = chrome_src self._SetupChecksum() diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index f7b78e39..c5c85942 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -35,6 +35,14 @@ PERF_RESULTS_FILE = 'perf-results.txt' CACHE_KEYS_FILE = 'cache_keys.txt' +class PidVerificationError(Exception): + """Error of perf PID verification in per-process mode.""" + + +class PerfDataReadError(Exception): + """Error of reading a perf.data header.""" + + class Result(object): """Class for holding the results of a single test run. @@ -58,6 +66,7 @@ class Result(object): self.results_file = [] self.turbostat_log_file = '' self.cpustats_log_file = '' + self.cpuinfo_file = '' self.top_log_file = '' self.wait_time_log_file = '' self.chrome_version = '' @@ -125,6 +134,13 @@ class Result(object): self.CopyFilesTo(dest_dir, self.results_file) self.CopyFilesTo(dest_dir, self.perf_data_files) self.CopyFilesTo(dest_dir, self.perf_report_files) + extra_files = [] + if self.top_log_file: + extra_files.append(self.top_log_file) + if self.cpuinfo_file: + extra_files.append(self.cpuinfo_file) + if extra_files: + self.CopyFilesTo(dest_dir, extra_files) if self.results_file or self.perf_data_files or self.perf_report_files: self._logger.LogOutput('Results files stored in %s.' % dest_dir) @@ -134,12 +150,12 @@ class Result(object): # while tast runs hold output under TEST_NAME/. # Both ensure to be unique. result_dir_name = self.test_name if self.suite == 'tast' else 'results' - results_dir = self.FindFilesInResultsDir( - '-name %s' % result_dir_name).split('\n')[0] + results_dir = self.FindFilesInResultsDir('-name %s' % + result_dir_name).split('\n')[0] if not results_dir: - self._logger.LogOutput( - 'WARNING: No results dir matching %r found' % result_dir_name) + self._logger.LogOutput('WARNING: No results dir matching %r found' % + result_dir_name) return self.CreateTarball(results_dir, tarball) @@ -231,8 +247,8 @@ class Result(object): command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) self.ce.RunCommand(command, print_to_console=False) - command = ('./generate_test_report --no-color --csv %s' % (os.path.join( - '/tmp', os.path.basename(self.temp_dir)))) + command = ('./generate_test_report --no-color --csv %s' % + (os.path.join('/tmp', os.path.basename(self.temp_dir)))) _, out, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command, print_to_console=False) keyvals_dict = {} @@ -349,6 +365,10 @@ class Result(object): """Get cpustats log path string.""" return self.FindFilesInResultsDir('-name cpustats.log').split('\n')[0] + def GetCpuinfoFile(self): + """Get cpustats log path string.""" + return self.FindFilesInResultsDir('-name cpuinfo.log').split('\n')[0] + def GetTopFile(self): """Get cpustats log path string.""" return self.FindFilesInResultsDir('-name top.log').split('\n')[0] @@ -378,8 +398,8 @@ class Result(object): perf_data_file) perf_report_file = '%s.report' % perf_data_file if os.path.exists(perf_report_file): - raise RuntimeError( - 'Perf report file already exists: %s' % perf_report_file) + raise RuntimeError('Perf report file already exists: %s' % + perf_report_file) chroot_perf_report_file = misc.GetInsideChrootPath( self.chromeos_root, perf_report_file) perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') @@ -417,8 +437,8 @@ class Result(object): if self.log_level != 'verbose': self._logger.LogOutput('Perf report generated successfully.') else: - raise RuntimeError( - 'Perf report not generated correctly. CMD: %s' % command) + raise RuntimeError('Perf report not generated correctly. CMD: %s' % + command) # Add a keyval to the dictionary for the events captured. perf_report_files.append( @@ -455,6 +475,7 @@ class Result(object): self.perf_report_files = self.GeneratePerfReportFiles() self.turbostat_log_file = self.GetTurbostatFile() self.cpustats_log_file = self.GetCpustatsFile() + self.cpuinfo_file = self.GetCpuinfoFile() self.top_log_file = self.GetTopFile() self.wait_time_log_file = self.GetWaitTimeFile() # TODO(asharif): Do something similar with perf stat. @@ -831,6 +852,88 @@ class Result(object): keyvals[key] = [result, unit] return keyvals + def ReadPidFromPerfData(self): + """Read PIDs from perf.data files. + + Extract PID from perf.data if "perf record" was running per process, + i.e. with "-p <PID>" and no "-a". + + Returns: + pids: list of PIDs. + + Raises: + PerfDataReadError when perf.data header reading fails. + """ + cmd = ['/usr/bin/perf', 'report', '--header-only', '-i'] + pids = [] + + for perf_data_path in self.perf_data_files: + perf_data_path_in_chroot = misc.GetInsideChrootPath( + self.chromeos_root, perf_data_path) + path_str = ' '.join(cmd + [perf_data_path_in_chroot]) + status, output, _ = self.ce.ChrootRunCommandWOutput( + self.chromeos_root, path_str) + if status: + # Error of reading a perf.data profile is fatal. + raise PerfDataReadError(f'Failed to read perf.data profile: {path_str}') + + # Pattern to search a line with "perf record" command line: + # # cmdline : /usr/bin/perf record -e instructions -p 123" + cmdline_regex = re.compile( + r'^\#\scmdline\s:\s+(?P<cmd>.*perf\s+record\s+.*)$') + # Pattern to search PID in a command line. + pid_regex = re.compile(r'^.*\s-p\s(?P<pid>\d+)\s*.*$') + for line in output.splitlines(): + cmd_match = cmdline_regex.match(line) + if cmd_match: + # Found a perf command line. + cmdline = cmd_match.group('cmd') + # '-a' is a system-wide mode argument. + if '-a' not in cmdline.split(): + # It can be that perf was attached to PID and was still running in + # system-wide mode. + # We filter out this case here since it's not per-process. + pid_match = pid_regex.match(cmdline) + if pid_match: + pids.append(pid_match.group('pid')) + # Stop the search and move to the next perf.data file. + break + else: + # cmdline wasn't found in the header. It's a fatal error. + raise PerfDataReadError(f'Perf command line is not found in {path_str}') + return pids + + def VerifyPerfDataPID(self): + """Verify PIDs in per-process perf.data profiles. + + Check that at list one top process is profiled if perf was running in + per-process mode. + + Raises: + PidVerificationError if PID verification of per-process perf.data profiles + fail. + """ + perf_data_pids = self.ReadPidFromPerfData() + if not perf_data_pids: + # In system-wide mode there are no PIDs. + self._logger.LogOutput('System-wide perf mode. Skip verification.') + return + + # PIDs will be present only in per-process profiles. + # In this case we need to verify that profiles are collected on the + # hottest processes. + top_processes = [top_cmd['cmd'] for top_cmd in self.top_cmds] + # top_process structure: <cmd>-<pid> + top_pids = [top_process.split('-')[-1] for top_process in top_processes] + for top_pid in top_pids: + if top_pid in perf_data_pids: + self._logger.LogOutput('PID verification passed! ' + f'Top process {top_pid} is profiled.') + return + raise PidVerificationError( + f'top processes {top_processes} are missing in perf.data traces with' + f' PID: {perf_data_pids}.') + def ProcessResults(self, use_cache=False): # Note that this function doesn't know anything about whether there is a # cache hit or miss. It should process results agnostic of the cache hit @@ -869,6 +972,9 @@ class Result(object): cpustats = self.ProcessCpustatsResults() if self.top_log_file: self.top_cmds = self.ProcessTopResults() + # Verify that PID in non system-wide perf.data and top_cmds are matching. + if self.perf_data_files and self.top_cmds: + self.VerifyPerfDataPID() if self.wait_time_log_file: with open(self.wait_time_log_file) as f: wait_time = f.readline().strip() @@ -949,8 +1055,8 @@ class Result(object): def CreateTarball(self, results_dir, tarball): if not results_dir.strip(): - raise ValueError( - 'Refusing to `tar` an empty results_dir: %r' % results_dir) + raise ValueError('Refusing to `tar` an empty results_dir: %r' % + results_dir) ret = self.ce.RunCommand('cd %s && ' 'tar ' @@ -1000,8 +1106,8 @@ class Result(object): if ret: command = 'rm -rf {0}'.format(temp_dir) self.ce.RunCommand(command) - raise RuntimeError( - 'Could not move dir %s to dir %s' % (temp_dir, cache_dir)) + raise RuntimeError('Could not move dir %s to dir %s' % + (temp_dir, cache_dir)) @classmethod def CreateFromRun(cls, diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index 2c1996c4..91ceed22 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -21,6 +21,8 @@ import test_flag from label import MockLabel from results_cache import CacheConditions +from results_cache import PerfDataReadError +from results_cache import PidVerificationError from results_cache import Result from results_cache import ResultsCache from results_cache import TelemetryResult @@ -158,6 +160,34 @@ keyvals = { 'b_string_strstr___abcdefghijklmnopqrstuvwxyz__': '0.0134553343333' } +PERF_DATA_HEADER = """ +# ======== +# captured on : Thu Jan 01 00:00:00 1980 +# header version : 1 +# data offset : 536 +# data size : 737678672 +# feat offset : 737679208 +# hostname : localhost +# os release : 5.4.61 +# perf version : +# arch : aarch64 +# nrcpus online : 8 +# nrcpus avail : 8 +# total memory : 5911496 kB +# cmdline : /usr/bin/perf record -e instructions -p {pid} +# event : name = instructions, , id = ( 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193 ), type = 8, size = 112 +# event : name = dummy:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9 +# CPU_TOPOLOGY info available, use -I to display +# pmu mappings: software = 1, uprobe = 6, cs_etm = 8, breakpoint = 5, tracepoint = 2, armv8_pmuv3 = 7 +# contains AUX area data (e.g. instruction trace) +# time of first sample : 0.000000 +# time of last sample : 0.000000 +# sample duration : 0.000 ms +# missing features: TRACING_DATA CPUDESC CPUID NUMA_TOPOLOGY BRANCH_STACK GROUP_DESC STAT CACHE MEM_TOPOLOGY CLOCKID DIR_FORMAT +# ======== +# +""" + TURBOSTAT_LOG_OUTPUT = \ """CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ CoreTmp - 329 12.13 2723 2393 10975 77 @@ -445,6 +475,7 @@ class ResultTest(unittest.TestCase): self.callGetTurbostatFile = False self.callGetCpustatsFile = False self.callGetTopFile = False + self.callGetCpuinfoFile = False self.callGetWaitTimeFile = False self.args = None self.callGatherPerfResults = False @@ -469,6 +500,7 @@ class ResultTest(unittest.TestCase): def setUp(self): self.result = Result(self.mock_logger, self.mock_label, 'average', self.mock_cmd_exec) + self.result.chromeos_root = '/tmp/chromeos' @mock.patch.object(os.path, 'isdir') @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') @@ -706,7 +738,8 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_chrootruncmd.call_count, 1) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], - ('/tmp', ('./generate_test_report --no-color --csv %s') % TMP_DIR1)) + (self.result.chromeos_root, + ('./generate_test_report --no-color --csv %s') % TMP_DIR1)) self.assertEqual(mock_getpath.call_count, 1) self.assertEqual(mock_mkdtemp.call_count, 1) self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) @@ -879,6 +912,15 @@ class ResultTest(unittest.TestCase): self.assertEqual(found_no_logs, '') @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') + def test_get_cpuinfo_file_finds_single_log(self, mock_runcmd): + """Expected behavior when a single cpuinfo file found.""" + self.result.results_dir = '/tmp/test_results' + self.result.ce.RunCommandWOutput = mock_runcmd + mock_runcmd.return_value = (0, 'some/long/path/cpuinfo.log', '') + found_single_log = self.result.GetCpuinfoFile() + self.assertEqual(found_single_log, 'some/long/path/cpuinfo.log') + + @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') def test_get_cpustats_file_finds_single_log(self, mock_runcmd): """Expected behavior when a single log file found.""" self.result.results_dir = '/tmp/test_results' @@ -906,6 +948,101 @@ class ResultTest(unittest.TestCase): found_no_logs = self.result.GetCpustatsFile() self.assertEqual(found_no_logs, '') + def test_verify_perf_data_pid_ok(self): + """Verify perf PID which is present in TOP_DATA.""" + self.result.top_cmds = TOP_DATA + # pid is present in TOP_DATA. + with mock.patch.object( + Result, 'ReadPidFromPerfData', return_value=['5713']): + self.result.VerifyPerfDataPID() + + def test_verify_perf_data_pid_fail(self): + """Test perf PID missing in top raises the error.""" + self.result.top_cmds = TOP_DATA + # pid is not in the list of top processes. + with mock.patch.object( + Result, 'ReadPidFromPerfData', return_value=['9999']): + with self.assertRaises(PidVerificationError): + self.result.VerifyPerfDataPID() + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_ok(self, mock_runcmd): + """Test perf header parser, normal flow.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + exp_pid = '12345' + mock_runcmd.return_value = (0, PERF_DATA_HEADER.format(pid=exp_pid), '') + pids = self.result.ReadPidFromPerfData() + self.assertEqual(pids, [exp_pid]) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_mult_profiles(self, mock_runcmd): + """Test multiple perf.data files with PID.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + # self.result.chromeos_root = '/tmp/chromeos' + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data.0', + '/tmp/chromeos/chroot/tmp/results/perf.data.1', + ] + # There is '-p <pid>' in command line but it's still system-wide: '-a'. + cmd_line = '# cmdline : /usr/bin/perf record -e instructions -p {pid}' + exp_perf_pids = ['1111', '2222'] + mock_runcmd.side_effect = [ + (0, cmd_line.format(pid=exp_perf_pids[0]), ''), + (0, cmd_line.format(pid=exp_perf_pids[1]), ''), + ] + pids = self.result.ReadPidFromPerfData() + self.assertEqual(pids, exp_perf_pids) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_no_pid(self, mock_runcmd): + """Test perf.data without PID.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + cmd_line = '# cmdline : /usr/bin/perf record -e instructions' + mock_runcmd.return_value = (0, cmd_line, '') + pids = self.result.ReadPidFromPerfData() + # pids is empty. + self.assertEqual(pids, []) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_system_wide(self, mock_runcmd): + """Test reading from system-wide profile with PID.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + # There is '-p <pid>' in command line but it's still system-wide: '-a'. + cmd_line = '# cmdline : /usr/bin/perf record -e instructions -a -p 1234' + mock_runcmd.return_value = (0, cmd_line, '') + pids = self.result.ReadPidFromPerfData() + # pids should be empty since it's not a per-process profiling. + self.assertEqual(pids, []) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_read_fail(self, mock_runcmd): + """Failure to read perf.data raises the error.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + # Error status of the profile read. + mock_runcmd.return_value = (1, '', '') + with self.assertRaises(PerfDataReadError): + self.result.ReadPidFromPerfData() + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_fail(self, mock_runcmd): + """Failure to find cmdline in perf.data header raises the error.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + # Empty output. + mock_runcmd.return_value = (0, '', '') + with self.assertRaises(PerfDataReadError): + self.result.ReadPidFromPerfData() + def test_process_turbostat_results_with_valid_data(self): """Normal case when log exists and contains valid data.""" self.result.turbostat_log_file = '/tmp/somelogfile.log' @@ -1102,10 +1239,11 @@ class ResultTest(unittest.TestCase): # Debug path not found self.result.label.debug_path = '' tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) + self.assertEqual(tmp, ['/tmp/chromeos/chroot%s' % fake_file]) self.assertEqual(mock_chrootruncmd.call_args_list[0][0], - ('/tmp', ('/usr/sbin/perf report -n ' - '-i %s --stdio > %s') % (fake_file, fake_file))) + (self.result.chromeos_root, + ('/usr/sbin/perf report -n ' + '-i %s --stdio > %s') % (fake_file, fake_file))) @mock.patch.object(misc, 'GetInsideChrootPath') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') @@ -1120,11 +1258,12 @@ class ResultTest(unittest.TestCase): # Debug path found self.result.label.debug_path = '/tmp/debug' tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) + self.assertEqual(tmp, ['/tmp/chromeos/chroot%s' % fake_file]) self.assertEqual(mock_chrootruncmd.call_args_list[0][0], - ('/tmp', ('/usr/sbin/perf report -n --symfs /tmp/debug ' - '--vmlinux /tmp/debug/boot/vmlinux ' - '-i %s --stdio > %s') % (fake_file, fake_file))) + (self.result.chromeos_root, + ('/usr/sbin/perf report -n --symfs /tmp/debug ' + '--vmlinux /tmp/debug/boot/vmlinux ' + '-i %s --stdio > %s') % (fake_file, fake_file))) @mock.patch.object(misc, 'GetOutsideChrootPath') def test_populate_from_run(self, mock_getpath): @@ -1157,6 +1296,10 @@ class ResultTest(unittest.TestCase): self.callGetTopFile = True return [] + def FakeGetCpuinfoFile(): + self.callGetCpuinfoFile = True + return [] + def FakeGetWaitTimeFile(): self.callGetWaitTimeFile = True return [] @@ -1169,7 +1312,6 @@ class ResultTest(unittest.TestCase): if mock_getpath: pass mock.get_path = '/tmp/chromeos/tmp/results_dir' - self.result.chromeos_root = '/tmp/chromeos' self.callGetResultsDir = False self.callGetResultsFile = False @@ -1178,6 +1320,7 @@ class ResultTest(unittest.TestCase): self.callGetTurbostatFile = False self.callGetCpustatsFile = False self.callGetTopFile = False + self.callGetCpuinfoFile = False self.callGetWaitTimeFile = False self.callProcessResults = False @@ -1188,6 +1331,7 @@ class ResultTest(unittest.TestCase): self.result.GetTurbostatFile = FakeGetTurbostatFile self.result.GetCpustatsFile = FakeGetCpustatsFile self.result.GetTopFile = FakeGetTopFile + self.result.GetCpuinfoFile = FakeGetCpuinfoFile self.result.GetWaitTimeFile = FakeGetWaitTimeFile self.result.ProcessResults = FakeProcessResults @@ -1200,6 +1344,7 @@ class ResultTest(unittest.TestCase): self.assertTrue(self.callGetTurbostatFile) self.assertTrue(self.callGetCpustatsFile) self.assertTrue(self.callGetTopFile) + self.assertTrue(self.callGetCpuinfoFile) self.assertTrue(self.callGetWaitTimeFile) self.assertTrue(self.callProcessResults) @@ -1436,8 +1581,7 @@ class ResultTest(unittest.TestCase): u'crypto-md5__crypto-md5': [10.5, u'ms'], u'string-tagcloud__string-tagcloud': [52.8, u'ms'], u'access-nbody__access-nbody': [8.5, u'ms'], - 'retval': - 0, + 'retval': 0, u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], u'math-cordic__math-cordic': [8.7, u'ms'], u'access-binary-trees__access-binary-trees': [4.5, u'ms'], @@ -1475,8 +1619,7 @@ class ResultTest(unittest.TestCase): u'crypto-md5__crypto-md5': [10.5, u'ms'], u'string-tagcloud__string-tagcloud': [52.8, u'ms'], u'access-nbody__access-nbody': [8.5, u'ms'], - 'retval': - 0, + 'retval': 0, u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], u'math-cordic__math-cordic': [8.7, u'ms'], u'access-binary-trees__access-binary-trees': [4.5, u'ms'], @@ -1705,8 +1848,9 @@ class TelemetryResultTest(unittest.TestCase): 'autotest_dir', 'debug_dir', '/tmp', 'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', False, None) - self.mock_machine = machine_manager.MockCrosMachine( - 'falco.cros', '/tmp/chromeos', 'average') + self.mock_machine = machine_manager.MockCrosMachine('falco.cros', + '/tmp/chromeos', + 'average') def test_populate_from_run(self): @@ -1786,10 +1930,12 @@ class ResultsCacheTest(unittest.TestCase): def FakeGetMachines(label): if label: pass - m1 = machine_manager.MockCrosMachine( - 'lumpy1.cros', self.results_cache.chromeos_root, 'average') - m2 = machine_manager.MockCrosMachine( - 'lumpy2.cros', self.results_cache.chromeos_root, 'average') + m1 = machine_manager.MockCrosMachine('lumpy1.cros', + self.results_cache.chromeos_root, + 'average') + m2 = machine_manager.MockCrosMachine('lumpy2.cros', + self.results_cache.chromeos_root, + 'average') return [m1, m2] mock_checksum.return_value = 'FakeImageChecksumabc123' @@ -1831,10 +1977,12 @@ class ResultsCacheTest(unittest.TestCase): def FakeGetMachines(label): if label: pass - m1 = machine_manager.MockCrosMachine( - 'lumpy1.cros', self.results_cache.chromeos_root, 'average') - m2 = machine_manager.MockCrosMachine( - 'lumpy2.cros', self.results_cache.chromeos_root, 'average') + m1 = machine_manager.MockCrosMachine('lumpy1.cros', + self.results_cache.chromeos_root, + 'average') + m2 = machine_manager.MockCrosMachine('lumpy2.cros', + self.results_cache.chromeos_root, + 'average') return [m1, m2] mock_checksum.return_value = 'FakeImageChecksumabc123' diff --git a/crosperf/results_report.py b/crosperf/results_report.py index ff6c4f96..dc80b53b 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -418,8 +418,8 @@ class TextResultsReport(ResultsReport): cpu_info = experiment.machine_manager.GetAllCPUInfo(experiment.labels) sections.append(self._MakeSection('CPUInfo', cpu_info)) - totaltime = ( - time.time() - experiment.start_time) if experiment.start_time else 0 + totaltime = (time.time() - + experiment.start_time) if experiment.start_time else 0 totaltime_str = 'Total experiment time:\n%d min' % (totaltime // 60) cooldown_waittime_list = ['Cooldown wait time:'] # When running experiment on multiple DUTs cooldown wait time may vary @@ -430,8 +430,9 @@ class TextResultsReport(ResultsReport): cooldown_waittime_list.append('DUT %s: %d min' % (dut, waittime // 60)) cooldown_waittime_str = '\n'.join(cooldown_waittime_list) sections.append( - self._MakeSection('Duration', '\n\n'.join( - [totaltime_str, cooldown_waittime_str]))) + self._MakeSection('Duration', + '\n\n'.join([totaltime_str, + cooldown_waittime_str]))) return '\n'.join(sections) @@ -505,7 +506,7 @@ class HTMLResultsReport(ResultsReport): experiment_file = '' if self.experiment is not None: experiment_file = self.experiment.experiment_file - # Use kwargs for sanity, and so that testing is a bit easier. + # Use kwargs for code readability, and so that testing is a bit easier. return templates.GenerateHTMLPage( perf_table=perf_table, chart_js=chart_javascript, diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py index e03ea431..1e96ef97 100755 --- a/crosperf/results_report_unittest.py +++ b/crosperf/results_report_unittest.py @@ -57,7 +57,8 @@ class FreeFunctionsTest(unittest.TestCase): ParseChromeosImage(os.path.dirname(buildbot_case)), ('', os.path.dirname(buildbot_img))) - # Ensure we don't act completely insanely given a few mildly insane paths. + # Ensure we do something reasonable when giving paths that don't quite + # match the expected pattern. fun_case = '/chromiumos_test_image.bin' self.assertEqual(ParseChromeosImage(fun_case), ('', fun_case)) diff --git a/debug_info_test/whitelist.py b/debug_info_test/allowlist.py index b53387a8..9cf42af0 100644 --- a/debug_info_test/whitelist.py +++ b/debug_info_test/allowlist.py @@ -3,12 +3,12 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Whitelist functions.""" +"""Allowlist functions.""" from __future__ import print_function -import os import glob +import os import re @@ -18,20 +18,20 @@ import re # # The performance bottleneck of this script is readelf. Unless this becomes # slower than readelf, don't waste time here. -def is_whitelisted(list_name, pattern): - """Check whether the given pattern is specified in the whitelist. +def is_allowlisted(list_name, pattern): + """Check whether the given pattern is specified in the allowlist. Args: - list_name: name of the whitelist. + list_name: name of the allowlist. pattern: the target string. Returns: True if matched otherwise False. """ - return pattern and whitelists[list_name].match(pattern) + return pattern and allowlists[list_name].match(pattern) -def prepare_whitelist(patterns): +def prepare_allowlist(patterns): """Join and compile the re patterns. Args: @@ -43,26 +43,26 @@ def prepare_whitelist(patterns): return re.compile('|'.join(patterns)) -def load_whitelists(dirname): - """Load whitelists under dirname. +def load_allowlists(dirname): + """Load allowlists under dirname. - A whitelist ends with .whitelist. + An allowlist ends with .allowlist. Args: dirname: path to the dir. Returns: - A dictionary of 'filename' -> whitelist matcher. + A dictionary of 'filename' -> allowlist matcher. """ wlist = {} - for fn in glob.glob(os.path.join(dirname, '*.whitelist')): + for fn in glob.glob(os.path.join(dirname, '*.allowlist')): key = os.path.splitext(os.path.basename(fn))[0] with open(fn, 'r', encoding='utf-8') as f: patterns = f.read().splitlines() patterns = [l for l in patterns if l != ''] patterns = [l for l in patterns if l[0] != '#'] - wlist[key] = prepare_whitelist(patterns) + wlist[key] = prepare_allowlist(patterns) return wlist -whitelists = load_whitelists(os.path.dirname(__file__)) +allowlists = load_allowlists(os.path.dirname(__file__)) diff --git a/debug_info_test/check_exist.py b/debug_info_test/check_exist.py index dbb89127..f2cc7c6b 100644 --- a/debug_info_test/check_exist.py +++ b/debug_info_test/check_exist.py @@ -10,7 +10,7 @@ from __future__ import print_function import os import subprocess -from whitelist import is_whitelisted +from allowlist import is_allowlisted def check_debug_info(dso_path, readelf_content): @@ -24,8 +24,8 @@ def check_debug_info(dso_path, readelf_content): True if debug info section exists, otherwise False. """ - # Return True if it is whitelisted - if is_whitelisted('exist_debug_info', dso_path): + # Return True if it is allowlisted + if is_allowlisted('exist_debug_info', dso_path): return True for l in readelf_content: @@ -46,8 +46,8 @@ def check_producer(dso_path, readelf_content): Notice: If no compile unit in DSO, also return True. """ - # Return True if it is whitelisted - if is_whitelisted('exist_producer', dso_path): + # Return True if it is allowlisted + if is_allowlisted('exist_producer', dso_path): return True # Indicate if there is a producer under each cu diff --git a/debug_info_test/check_ngcc.py b/debug_info_test/check_ngcc.py index 501bb988..c86c220a 100644 --- a/debug_info_test/check_ngcc.py +++ b/debug_info_test/check_ngcc.py @@ -7,7 +7,7 @@ from __future__ import print_function -from whitelist import is_whitelisted +from allowlist import is_allowlisted def not_by_gcc(dso_path, producer, comp_path): @@ -21,10 +21,10 @@ def not_by_gcc(dso_path, producer, comp_path): Returns: False if compiled by gcc otherwise True. """ - if is_whitelisted('ngcc_comp_path', comp_path): + if is_allowlisted('ngcc_comp_path', comp_path): return True - if is_whitelisted('ngcc_dso_path', dso_path): + if is_allowlisted('ngcc_dso_path', dso_path): return True return 'GNU C' not in producer diff --git a/debug_info_test/exist_debug_info.whitelist b/debug_info_test/exist_debug_info.allowlist index e0076fd6..e0076fd6 100644 --- a/debug_info_test/exist_debug_info.whitelist +++ b/debug_info_test/exist_debug_info.allowlist diff --git a/debug_info_test/exist_producer.whitelist b/debug_info_test/exist_producer.allowlist index ee75de72..ee75de72 100644 --- a/debug_info_test/exist_producer.whitelist +++ b/debug_info_test/exist_producer.allowlist diff --git a/debug_info_test/ngcc_comp_path.whitelist b/debug_info_test/ngcc_comp_path.allowlist index 45c5b4a2..45c5b4a2 100644 --- a/debug_info_test/ngcc_comp_path.whitelist +++ b/debug_info_test/ngcc_comp_path.allowlist diff --git a/debug_info_test/ngcc_dso_path.whitelist b/debug_info_test/ngcc_dso_path.allowlist index 8d63a521..858465e0 100644 --- a/debug_info_test/ngcc_dso_path.whitelist +++ b/debug_info_test/ngcc_dso_path.allowlist @@ -1,5 +1,5 @@ # DSOs specified here are not CrOS packages compiled within CrOS SDK. -# CrOS packages should be whitelisted in *_comp_path.whitelist +# CrOS packages should be allowlisted in *_comp_path.allowlist # modules we don't care: .*/binutils/.* .*/binutils-bin/.* diff --git a/llvm_tools/README.md b/llvm_tools/README.md index 1c1862d8..783ec22d 100644 --- a/llvm_tools/README.md +++ b/llvm_tools/README.md @@ -278,6 +278,7 @@ $ ./auto_llvm_bisection.py --start_rev 369410 --end_rev 369420 \ --last_tested /abs/path/to/last_tested_file.json \ --extra_change_lists 513590 1394249 \ --options latest-toolchain nochromesdk \ + --chroot_path /path/to/chromeos/chroot \ --builder eve-release-tryjob ``` @@ -488,7 +489,7 @@ these synthesized numbers and git SHAs. Usage should be straightforward: 6f635f90929da9545dd696071a829a1a42f84b30 ~> ./git_llvm_rev.py --llvm_dir llvm-project-copy/ --sha 6f635f90929da9545dd696071a829a1a42f84b30 r380000 -~> ./git_llvm_rev.py --llvm_dir llvm-project-copy/ --sha origin/master +~> ./git_llvm_rev.py --llvm_dir llvm-project-copy/ --sha origin/some-branch r387778 ``` @@ -550,3 +551,44 @@ PYTHONPATH=../ ./nightly_revert_checker.py \ --llvm_dir llvm-project-copy \ --chromeos_dir ../../../../ ``` + +### `bisect_clang_crashes.py` + +This script downloads clang crash diagnoses from +gs://chromeos-toolchain-artifacts/clang-crash-diagnoses and sends them to 4c for +bisection. + +Usage example: + +``` +$ ./bisect_clang_crashes.py --4c 4c-cli --state_file ./output/state.json +``` + +The above command downloads the artifacts of clang crash diagnoses and send them +to 4c server for bisection. The summary of submitted jobs will be saved in +output/state.json under the current path. The output directory will be created +automatically if it does not exist yet. To get more information of the submitted +jobs, please refer to go/4c-cli. + +### `upload_lexan_crashes_to_forcey.py` + +This script downloads clang crash diagnoses from Lexan's bucket and sends them +to 4c for bisection. + +Usage example: + +``` +$ ./upload_lexan_crashes_to_forcey.py --4c 4c-cli \ + --state_file ./output/state.json +``` + +The above command downloads the artifacts of clang crash diagnoses and send them +to 4c server for bisection. The summary of submitted jobs will be saved in +output/state.json under the current path. The output directory will be created +automatically if it does not exist yet. To get more information of the submitted +jobs, please refer to go/4c-cli. + +Note that it's recommended to 'seed' the state file with a most recent upload +date. This can be done by running this tool *once* with a `--last_date` flag. +This flag has the script override whatever's in the state file (if anything) and +start submitting all crashes uploaded starting at the given day. diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py new file mode 100755 index 00000000..e8ee2ab6 --- /dev/null +++ b/llvm_tools/bisect_clang_crashes.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Fetches and submits the artifacts from Chrome OS toolchain's crash bucket. +""" + +# pylint: disable=cros-logging-import + +import argparse +import glob +import json +import logging +import os +import os.path +import shutil +import subprocess +import sys + +import chroot + + +def get_artifacts(pattern): + results = subprocess.check_output(['gsutil.py', 'ls', pattern], + stderr=subprocess.STDOUT, + encoding='utf-8') + return sorted(l.strip() for l in results.splitlines()) + + +def get_crash_reproducers(working_dir): + results = [] + for src in [ + f for f in glob.glob('%s/*.c*' % working_dir) + if f.split('.')[-1] in ['c', 'cc', 'cpp'] + ]: + script = '.'.join(src.split('.')[:-1]) + '.sh' + if not os.path.exists(script): + logging.warning('could not find the matching script of %s', src) + else: + results.append((src, script)) + return results + + +def submit_crash_to_forcey(forcey: str, temporary_directory: str, + buildbucket_id: str, url: str) -> None: + dest_dir = os.path.join(temporary_directory, buildbucket_id) + dest_file = os.path.join(dest_dir, os.path.basename(url)) + logging.info('Downloading and submitting %r...', url) + subprocess.check_output(['gsutil.py', 'cp', url, dest_file], + stderr=subprocess.STDOUT) + subprocess.check_output(['tar', '-xJf', dest_file], cwd=dest_dir) + for src, script in get_crash_reproducers(dest_dir): + subprocess.check_output([ + forcey, 'reduce', '-wait=false', '-note', + '%s:%s' % (url, src), '-sh_file', script, '-src_file', src + ]) + + +def main(argv): + chroot.VerifyOutsideChroot() + logging.basicConfig( + format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s', + level=logging.INFO, + ) + cur_dir = os.path.dirname(os.path.abspath(__file__)) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--4c', dest='forcey', required=True, help='Path to a 4c client binary') + parser.add_argument( + '--state_file', + default=os.path.join(cur_dir, 'chromeos-state.json'), + help='The path to the state file.') + parser.add_argument( + '--nocleanup', + action='store_false', + dest='cleanup', + help='Keep temporary files created after the script finishes.') + opts = parser.parse_args(argv) + + state_file = os.path.abspath(opts.state_file) + os.makedirs(os.path.dirname(state_file), exist_ok=True) + temporary_directory = '/tmp/bisect_clang_crashes' + os.makedirs(temporary_directory, exist_ok=True) + urls = get_artifacts('gs://chromeos-toolchain-artifacts/clang-crash-diagnoses' + '/**/*clang_crash_diagnoses.tar.xz') + logging.info('%d crash URLs found', len(urls)) + + visited = {} + if os.path.exists(state_file): + buildbucket_ids = {url.split('/')[-2] for url in urls} + with open(state_file, encoding='utf-8') as f: + data = json.load(f) + visited = {k: v for k, v in data.items() if k in buildbucket_ids} + logging.info('Successfully loaded %d previously-submitted crashes', + len(visited)) + + try: + for url in urls: + splits = url.split('/') + buildbucket_id = splits[-2] + # Skip the builds that has been processed + if buildbucket_id in visited: + continue + submit_crash_to_forcey( + forcey=opts.forcey, + temporary_directory=temporary_directory, + buildbucket_id=buildbucket_id, + url=url, + ) + visited[buildbucket_id] = url + + exception_in_flight = False + except: + exception_in_flight = True + raise + finally: + if exception_in_flight: + # This is best-effort. If the machine powers off or similar, we'll just + # resubmit the same crashes, which is suboptimal, but otherwise + # acceptable. + logging.error('Something went wrong; attempting to save our work...') + else: + logging.info('Persisting state...') + + tmp_state_file = state_file + '.tmp' + with open(tmp_state_file, 'w', encoding='utf-8') as f: + json.dump(visited, f, indent=2) + os.rename(tmp_state_file, state_file) + + logging.info('State successfully persisted') + + if opts.cleanup: + shutil.rmtree(temporary_directory) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/llvm_tools/bisect_clang_crashes_unittest.py b/llvm_tools/bisect_clang_crashes_unittest.py new file mode 100755 index 00000000..c9143450 --- /dev/null +++ b/llvm_tools/bisect_clang_crashes_unittest.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for bisect_clang_crashes.""" + +# pylint: disable=cros-logging-import +import glob +import logging +import os.path +import subprocess +import unittest +import unittest.mock as mock + +import bisect_clang_crashes + + +class Test(unittest.TestCase): + """Tests for bisect_clang_crashes.""" + + class _SilencingFilter(object): + """Silences all log messages. + + Also collects info about log messages that would've been emitted. + """ + + def __init__(self): + self.messages = [] + + def filter(self, record): + self.messages.append(record.getMessage()) + return 0 + + @mock.patch.object(subprocess, 'check_output') + def test_get_artifacts(self, mock_gsutil_ls): + pattern = 'gs://chromeos-toolchain-artifacts/clang-crash-diagnoses/' \ + '**/*clang_crash_diagnoses.tar.xz' + mock_gsutil_ls.return_value = 'artifact1\nartifact2\nartifact3' + results = bisect_clang_crashes.get_artifacts(pattern) + self.assertEqual(results, ['artifact1', 'artifact2', 'artifact3']) + mock_gsutil_ls.assert_called_once_with(['gsutil.py', 'ls', pattern], + stderr=subprocess.STDOUT, + encoding='utf-8') + + @mock.patch.object(os.path, 'exists') + @mock.patch.object(glob, 'glob') + def test_get_crash_reproducers_succeed(self, mock_file_search, + mock_file_check): + working_dir = 'SomeDirectory' + mock_file_search.return_value = ['a.c', 'b.cpp', 'c.cc'] + mock_file_check.side_effect = [True, True, True] + results = bisect_clang_crashes.get_crash_reproducers(working_dir) + mock_file_search.assert_called_once_with('%s/*.c*' % working_dir) + self.assertEqual(mock_file_check.call_count, 3) + self.assertEqual(mock_file_check.call_args_list[0], mock.call('a.sh')) + self.assertEqual(mock_file_check.call_args_list[1], mock.call('b.sh')) + self.assertEqual(mock_file_check.call_args_list[2], mock.call('c.sh')) + self.assertEqual(results, [('a.c', 'a.sh'), ('b.cpp', 'b.sh'), + ('c.cc', 'c.sh')]) + + @mock.patch.object(os.path, 'exists') + @mock.patch.object(glob, 'glob') + def test_get_crash_reproducers_no_matching_script(self, mock_file_search, + mock_file_check): + + def silence_logging(): + root = logging.getLogger() + filt = self._SilencingFilter() + root.addFilter(filt) + self.addCleanup(root.removeFilter, filt) + return filt + + log_filter = silence_logging() + working_dir = 'SomeDirectory' + mock_file_search.return_value = ['a.c', 'b.cpp', 'c.cc'] + mock_file_check.side_effect = [True, False, True] + results = bisect_clang_crashes.get_crash_reproducers(working_dir) + mock_file_search.assert_called_once_with('%s/*.c*' % working_dir) + self.assertEqual(mock_file_check.call_count, 3) + self.assertEqual(mock_file_check.call_args_list[0], mock.call('a.sh')) + self.assertEqual(mock_file_check.call_args_list[1], mock.call('b.sh')) + self.assertEqual(mock_file_check.call_args_list[2], mock.call('c.sh')) + self.assertEqual(results, [('a.c', 'a.sh'), ('c.cc', 'c.sh')]) + self.assertTrue( + any('could not find the matching script of b.cpp' in x + for x in log_filter.messages), log_filter.messages) + + +if __name__ == '__main__': + unittest.main() diff --git a/llvm_tools/cherrypick_cl.py b/llvm_tools/cherrypick_cl.py index 78330d19..9e306725 100755 --- a/llvm_tools/cherrypick_cl.py +++ b/llvm_tools/cherrypick_cl.py @@ -170,32 +170,34 @@ def main(): help='Automatically create a CL if specified') args = parser.parse_args() - llvm_config = git_llvm_rev.LLVMConfig( - remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) - llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( args.chroot_path, chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm']))[0] + llvm_symlink_dir = os.path.dirname(llvm_symlink) + + git_status = subprocess.check_output(['git', 'status', '-s'], + cwd=llvm_symlink_dir, + encoding='utf-8') + if git_status: + raise ValueError('Uncommited changes detected in %s' % + os.path.dirname(os.path.dirname(llvm_symlink_dir))) + start_sha = args.start_sha if start_sha == 'llvm': - start_sha = parse_ebuild_for_assignment( - os.path.dirname(llvm_symlink), 'LLVM_HASH') + start_sha = parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_HASH') elif start_sha == 'llvm-next': - start_sha = parse_ebuild_for_assignment( - os.path.dirname(llvm_symlink), 'LLVM_NEXT_HASH') + start_sha = parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_NEXT_HASH') logging.info('Base llvm hash == %s', start_sha) + llvm_config = git_llvm_rev.LLVMConfig( + remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) + start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) start_rev = git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha) if args.create_cl: branch = 'cherry-pick' - symlink = os.path.dirname( - chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm'])[0]) - symlink = chroot.ConvertChrootPathsToAbsolutePaths(args.chroot_path, - [symlink])[0] - symlink_dir = os.path.dirname(symlink) - git.CreateBranch(symlink_dir, branch) + git.CreateBranch(llvm_symlink_dir, branch) symlinks_to_uprev = [] commit_messages = [ 'llvm: cherry-pick CLs from upstream\n', @@ -221,9 +223,9 @@ def main(): relative_patches_dir = 'cherry' if package == 'llvm' else '' patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) logging.info('Cherrypicking %s (%s) into %s', rev, sha, package) + add_cherrypick(patches_json_path, patches_dir, relative_patches_dir, start_rev, llvm_config.dir, rev, sha, package) - if args.create_cl: symlinks_to_uprev.extend(symlinks) commit_messages.extend([ @@ -240,8 +242,8 @@ def main(): for symlink in symlinks_to_uprev: update_chromeos_llvm_hash.UprevEbuildSymlink(symlink) subprocess.check_output(['git', 'add', '--all'], cwd=symlink_dir) - git.UploadChanges(symlink_dir, branch, commit_messages) - git.DeleteBranch(symlink_dir, branch) + git.UploadChanges(llvm_symlink_dir, branch, commit_messages) + git.DeleteBranch(llvm_symlink_dir, branch) if __name__ == '__main__': diff --git a/llvm_tools/fetch_cros_sdk_rolls.py b/llvm_tools/fetch_cros_sdk_rolls.py new file mode 100755 index 00000000..42af678a --- /dev/null +++ b/llvm_tools/fetch_cros_sdk_rolls.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Gets info about completed chromiumos-sdk runs. + +Moreover, this script exists to get versions of published sdk tarballs in +gs://chromiumos-sdk/. The hope is that it'll help answer the question "when did +the toolchain ebuild ${x} go live?" +""" + +# pylint: disable=cros-logging-import + +import argparse +import json +import logging +import os +import shutil +import subprocess +import sys +import tempfile +from typing import Dict, List +from pathlib import Path + + +def fetch_all_sdk_manifest_paths() -> List[str]: + """Fetches all paths of SDK manifests; newer = later in the return value.""" + results = subprocess.run( + ['gsutil', 'ls', 'gs://chromiumos-sdk/cros-sdk-20??.*.Manifest'], + check=True, + stdout=subprocess.PIPE, + encoding='utf-8', + ).stdout + # These are named so that sorted order == newest last. + return sorted(x.strip() for x in results.splitlines()) + + +def fetch_manifests_into(into_dir: Path, manifests: List[str]): + # Wrap this in a `try` block because gsutil likes to print to stdout *and* + # stderr even on success, so we silence them & only print on failure. + try: + subprocess.run( + [ + 'gsutil', + '-m', + 'cp', + '-I', + str(into_dir), + ], + check=True, + input='\n'.join(manifests), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding='utf-8', + ) + except subprocess.CalledProcessError as e: + logging.exception('gsutil failed; output:\n%s', e.stdout) + + +def load_manifest_versions(manifest: Path) -> Dict[str, str]: + with manifest.open(encoding='utf-8') as f: + raw_versions = json.load(f) + + # We get a dict of list of lists of versions and some other metadata, e.g. + # {"foo/bar": [["1.2.3", {}]]} + # Trim out the metadata. + return {k: v[0][0] for k, v in raw_versions['packages'].items()} + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '-d', '--debug', action='store_true', help='Emit debugging output') + parser.add_argument( + '-n', + '--number', + type=int, + default=20, + help='Number of recent manifests to fetch info about. 0 means unlimited.') + args = parser.parse_args() + + is_debug = args.debug + logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO) + + logging.debug('Fetching SDK manifests') + manifest_paths = fetch_all_sdk_manifest_paths() + logging.debug('%d SDK manifests fetched', len(manifest_paths)) + + number = args.number + if number: + manifest_paths = manifest_paths[-number:] + + tempdir = Path(tempfile.mkdtemp(prefix='cros-sdk-rolls')) + try: + logging.debug('Working in tempdir %r', tempdir) + fetch_manifests_into(tempdir, manifest_paths) + + for path in manifest_paths: + basename = os.path.basename(path) + versions = load_manifest_versions(tempdir.joinpath(basename)) + print(f'{basename}: {versions["sys-devel/llvm"]}') + finally: + if is_debug: + logging.debug('Keeping around tempdir %r to aid debugging', tempdir) + else: + shutil.rmtree(tempdir) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py index a5b5429e..4c479962 100755 --- a/llvm_tools/get_llvm_hash.py +++ b/llvm_tools/get_llvm_hash.py @@ -17,7 +17,8 @@ import tempfile from contextlib import contextmanager import git_llvm_rev -from subprocess_helpers import CheckCommand, check_output +from subprocess_helpers import CheckCommand +from subprocess_helpers import check_output _LLVM_GIT_URL = ('https://chromium.googlesource.com/external/github.com/llvm' '/llvm-project') @@ -39,7 +40,7 @@ def GetVersionFrom(src_dir, git_hash): version = git_llvm_rev.translate_sha_to_rev( git_llvm_rev.LLVMConfig(remote='origin', dir=src_dir), git_hash) # Note: branches aren't supported - assert version.branch == 'master', version.branch + assert version.branch == git_llvm_rev.MAIN_BRANCH, version.branch return version.number @@ -59,7 +60,7 @@ def GetGitHashFrom(src_dir, version): return git_llvm_rev.translate_rev_to_sha( git_llvm_rev.LLVMConfig(remote='origin', dir=src_dir), - git_llvm_rev.Rev(branch='master', number=version)) + git_llvm_rev.Rev(branch=git_llvm_rev.MAIN_BRANCH, number=version)) @contextmanager @@ -85,13 +86,10 @@ def CreateTempLLVMRepo(temp_dir): """ abs_path_to_llvm_project_dir = GetAndUpdateLLVMProjectInLLVMTools() - - add_worktree_cmd = [ + CheckCommand([ 'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'add', '--detach', - temp_dir, 'master' - ] - - CheckCommand(add_worktree_cmd) + temp_dir, git_llvm_rev.MAIN_BRANCH + ]) try: yield temp_dir @@ -117,7 +115,7 @@ def GetAndUpdateLLVMProjectInLLVMTools(): Raises: ValueError: LLVM repo (in 'llvm-project-copy' dir.) has changes or failed to - checkout to master or failed to fetch from chromium mirror of LLVM. + checkout to main or failed to fetch from chromium mirror of LLVM. """ abs_path_to_llvm_tools_dir = os.path.dirname(os.path.abspath(__file__)) @@ -127,7 +125,8 @@ def GetAndUpdateLLVMProjectInLLVMTools(): if not os.path.isdir(abs_path_to_llvm_project_dir): print( - 'Checking out LLVM from scratch. This could take a while...', + 'Checking out LLVM from scratch. This could take a while...\n' + '(This should only need to be done once, though.)', file=sys.stderr) os.mkdir(abs_path_to_llvm_project_dir) @@ -143,15 +142,11 @@ def GetAndUpdateLLVMProjectInLLVMTools(): raise ValueError('LLVM repo in %s has changes, please remove.' % abs_path_to_llvm_project_dir) - checkout_to_master_cmd = [ - 'git', '-C', abs_path_to_llvm_project_dir, 'checkout', 'master' - ] - - CheckCommand(checkout_to_master_cmd) - - update_master_cmd = ['git', '-C', abs_path_to_llvm_project_dir, 'pull'] - - CheckCommand(update_master_cmd) + CheckCommand([ + 'git', '-C', abs_path_to_llvm_project_dir, 'checkout', + git_llvm_rev.MAIN_BRANCH + ]) + CheckCommand(['git', '-C', abs_path_to_llvm_project_dir, 'pull']) return abs_path_to_llvm_project_dir @@ -298,14 +293,9 @@ class LLVMHash(object): def GetTopOfTrunkGitHash(self): """Gets the latest git hash from top of trunk of LLVM.""" - path_to_master_branch = 'refs/heads/master' - - llvm_tot_git_hash_cmd = [ - 'git', 'ls-remote', _LLVM_GIT_URL, path_to_master_branch - ] - - llvm_tot_git_hash = check_output(llvm_tot_git_hash_cmd) - + path_to_main_branch = 'refs/heads/master' + llvm_tot_git_hash = check_output( + ['git', 'ls-remote', _LLVM_GIT_URL, path_to_main_branch]) return llvm_tot_git_hash.rstrip().split()[0] diff --git a/llvm_tools/get_llvm_hash_unittest.py b/llvm_tools/get_llvm_hash_unittest.py index c828f433..2e56aed5 100755 --- a/llvm_tools/get_llvm_hash_unittest.py +++ b/llvm_tools/get_llvm_hash_unittest.py @@ -8,12 +8,11 @@ from __future__ import print_function -import get_llvm_hash import subprocess import unittest import unittest.mock as mock -import test_helpers +import get_llvm_hash from get_llvm_hash import LLVMHash # We grab protected stuff from get_llvm_hash. That's OK. @@ -87,7 +86,7 @@ class TestGetLLVMHash(unittest.TestCase): @mock.patch.object(subprocess, 'check_output') def testSuccessfullyGetGitHashFromToTOfLLVM(self, mock_check_output): - mock_check_output.return_value = 'a123testhash1 path/to/master\n' + mock_check_output.return_value = 'a123testhash1 path/to/main\n' self.assertEqual(LLVMHash().GetTopOfTrunkGitHash(), 'a123testhash1') mock_check_output.assert_called_once() diff --git a/llvm_tools/git_llvm_rev.py b/llvm_tools/git_llvm_rev.py index 8eefcdce..07209f1e 100755 --- a/llvm_tools/git_llvm_rev.py +++ b/llvm_tools/git_llvm_rev.py @@ -18,6 +18,8 @@ import subprocess import sys import typing as t +MAIN_BRANCH = 'master' + # Note that after base_llvm_sha, we reach The Wild West(TM) of commits. # So reasonable input that could break us includes: # @@ -52,9 +54,9 @@ class Rev(t.NamedTuple('Rev', (('branch', str), ('number', int)))): # pairs. # # We support r${commits_since_base_commit} as shorthand for - # (master, r${commits_since_base_commit}). + # (main, r${commits_since_base_commit}). if rev.startswith('r'): - branch_name = 'master' + branch_name = MAIN_BRANCH rev_string = rev[1:] else: match = re.match(r'\((.+), r(\d+)\)', rev) @@ -67,7 +69,7 @@ class Rev(t.NamedTuple('Rev', (('branch', str), ('number', int)))): def __str__(self) -> str: branch_name, number = self - if branch_name == 'master': + if branch_name == MAIN_BRANCH: return 'r%d' % number return '(%s, r%d)' % (branch_name, number) @@ -141,7 +143,7 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev: cwd=llvm_config.dir, ) count = int(result.strip()) - return Rev(branch='master', number=count + base_llvm_revision) + return Rev(branch=MAIN_BRANCH, number=count + base_llvm_revision) # Otherwise, either: # - |merge_base| is |sha| (we have a guaranteed llvm-svn number on |sha|) @@ -150,7 +152,7 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev: merge_base_number = translate_prebase_sha_to_rev_number( llvm_config, merge_base) if merge_base == sha: - return Rev(branch='master', number=merge_base_number) + return Rev(branch=MAIN_BRANCH, number=merge_base_number) distance_from_base = check_output( [ @@ -270,7 +272,7 @@ def translate_rev_to_sha(llvm_config: LLVMConfig, rev: Rev) -> str: """ branch, number = rev - if branch == 'master': + if branch == MAIN_BRANCH: if number < base_llvm_revision: return translate_prebase_rev_to_sha(llvm_config, rev) base_sha = base_llvm_sha diff --git a/llvm_tools/git_llvm_rev_test.py b/llvm_tools/git_llvm_rev_test.py index 1e38f589..74280c5d 100755 --- a/llvm_tools/git_llvm_rev_test.py +++ b/llvm_tools/git_llvm_rev_test.py @@ -6,12 +6,11 @@ """Tests for git_llvm_rev.""" -from __future__ import print_function - import unittest import git_llvm_rev import llvm_project +from git_llvm_rev import MAIN_BRANCH def get_llvm_config() -> git_llvm_rev.LLVMConfig: @@ -32,17 +31,17 @@ class Test(unittest.TestCase): def test_sha_to_rev_on_base_sha_works(self) -> None: sha = self.rev_to_sha_with_round_trip( git_llvm_rev.Rev( - branch='master', number=git_llvm_rev.base_llvm_revision)) + branch=MAIN_BRANCH, number=git_llvm_rev.base_llvm_revision)) self.assertEqual(sha, git_llvm_rev.base_llvm_sha) def test_sha_to_rev_prior_to_base_rev_works(self) -> None: sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=375000)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=375000)) self.assertEqual(sha, '2f6da767f13b8fd81f840c211d405fea32ac9db7') def test_sha_to_rev_after_base_rev_works(self) -> None: sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=375506)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=375506)) self.assertEqual(sha, '3bf7fddeb05655d9baed4cc69e13535c677ed1dd') def test_llvm_svn_parsing_runs_ignore_reverts(self) -> None: @@ -51,18 +50,19 @@ class Test(unittest.TestCase): # Commit which performed the revert sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=374895)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=374895)) self.assertEqual(sha, '1731fc88d1fa1fa55edd056db73a339b415dd5d6') # Commit that was reverted sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=374841)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=374841)) self.assertEqual(sha, '2a1386c81de504b5bda44fbecf3f7b4cdfd748fc') def test_imaginary_revs_raise(self) -> None: with self.assertRaises(ValueError) as r: git_llvm_rev.translate_rev_to_sha( - get_llvm_config(), git_llvm_rev.Rev(branch='master', number=9999999)) + get_llvm_config(), + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=9999999)) self.assertIn('Try updating your tree?', str(r.exception)) @@ -71,15 +71,15 @@ class Test(unittest.TestCase): # properties about it. merge_sha_rev_number = 4496 + git_llvm_rev.base_llvm_revision sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=merge_sha_rev_number)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=merge_sha_rev_number)) self.assertEqual(sha, '0f0d0ed1c78f1a80139a1f2133fad5284691a121') sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=merge_sha_rev_number - 1)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=merge_sha_rev_number - 1)) self.assertEqual(sha, '6f635f90929da9545dd696071a829a1a42f84b30') sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=merge_sha_rev_number + 1)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=merge_sha_rev_number + 1)) self.assertEqual(sha, '199700a5cfeedf227619f966aa3125cef18bc958') # NOTE: The below tests have _zz_ in their name as an optimization. Iterating @@ -101,11 +101,11 @@ class Test(unittest.TestCase): backing_sha = 'c89a3d78f43d81b9cff7b9248772ddf14d21b749' sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev(branch='master', number=rev_number)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, number=rev_number)) self.assertEqual(sha, backing_sha) - # Note that this won't round-trip: since this commit is on the master - # branch, we'll pick master for this. That's fine + # Note that this won't round-trip: since this commit is on the main + # branch, we'll pick main for this. That's fine. sha = git_llvm_rev.translate_rev_to_sha( get_llvm_config(), git_llvm_rev.Rev(branch='release/9.x', number=rev_number)) @@ -113,7 +113,7 @@ class Test(unittest.TestCase): def test_zz_branch_revs_work_after_merge_points(self) -> None: # Picking the commit on the 9.x branch after the merge-base for that + - # master. Note that this is where llvm-svn numbers should diverge from + # main. Note that this is where llvm-svn numbers should diverge from # ours, and are therefore untrustworthy. The commit for this *does* have a # different `llvm-svn:` string than we should have. sha = self.rev_to_sha_with_round_trip( diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index 2772ca48..c8d694cd 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -13,10 +13,12 @@ import enum import errno import json import os +import subprocess import sys import chroot import get_llvm_hash +import git_llvm_rev import modify_a_tryjob import update_tryjob_status @@ -28,11 +30,6 @@ class BisectionExitStatus(enum.Enum): BISECTION_COMPLETE = 126 -def is_file_and_json(json_file): - """Validates that the file exists and is a JSON file.""" - return os.path.isfile(json_file) and json_file.endswith('.json') - - def GetCommandLineArgs(): """Parses the command line for the command line arguments.""" @@ -121,30 +118,27 @@ def GetCommandLineArgs(): help='display contents of a command to the terminal ' '(default: %(default)s)') + # Add argument for whether to display command contents to `stdout`. + parser.add_argument( + '--nocleanup', + action='store_false', + dest='cleanup', + help='Abandon CLs created for bisectoin') + args_output = parser.parse_args() assert args_output.start_rev < args_output.end_rev, ( - 'Start revision %d is >= end revision %d' % (args_output.start_rev, - args_output.end_rev)) + 'Start revision %d is >= end revision %d' % + (args_output.start_rev, args_output.end_rev)) if args_output.last_tested and not args_output.last_tested.endswith('.json'): - raise ValueError( - 'Filed provided %s does not end in ".json"' % args_output.last_tested) + raise ValueError('Filed provided %s does not end in ".json"' % + args_output.last_tested) return args_output -def _ValidateStartAndEndAgainstJSONStartAndEnd(start, end, json_start, - json_end): - """Valides that the command line arguments are the same as the JSON.""" - - if start != json_start or end != json_end: - raise ValueError('The start %d or the end %d version provided is ' - 'different than "start" %d or "end" %d in the .JSON ' - 'file' % (start, end, json_start, json_end)) - - -def GetStartAndEndRevision(start, end, tryjobs): +def GetRemainingRange(start, end, tryjobs): """Gets the start and end intervals in 'json_file'. Args: @@ -230,145 +224,56 @@ def GetStartAndEndRevision(start, end, tryjobs): return good_rev, bad_rev, pending_revisions, skip_revisions -def GetRevisionsBetweenBisection(start, end, parallel, src_path, - pending_revisions, skip_revisions): - """Gets the revisions between 'start' and 'end'. - - Sometimes, the LLVM source tree's revisions do not increment by 1 (there is - a jump), so need to construct a list of all revisions that are NOT missing - between 'start' and 'end'. Then, the step amount (i.e. length of the list - divided by ('parallel' + 1)) will be used for indexing into the list. - - Args: - start: The start revision. - end: The end revision. - parallel: The number of tryjobs to create between 'start' and 'end'. - src_path: The absolute path to the LLVM source tree to use. - pending_revisions: A set containing 'pending' revisions that are between - 'start' and 'end'. - skip_revisions: A set containing revisions between 'start' and 'end' that - are to be skipped. - - Returns: - A list of revisions between 'start' and 'end'. - """ - - valid_revisions = [] - - # Start at ('start' + 1) because 'start' is the good revision. - # - # FIXME: Searching for each revision from ('start' + 1) up to 'end' in the - # LLVM source tree is a quadratic algorithm. It's a good idea to optimize - # this. - for cur_revision in range(start + 1, end): - try: - if cur_revision not in pending_revisions and \ - cur_revision not in skip_revisions: - # Verify that the current revision exists by finding its corresponding - # git hash in the LLVM source tree. - get_llvm_hash.GetGitHashFrom(src_path, cur_revision) - valid_revisions.append(cur_revision) - except ValueError: - # Could not find the git hash for the current revision. - continue - - # ('parallel' + 1) so that the last revision in the list is not close to - # 'end' (have a bit more coverage). - index_step = len(valid_revisions) // (parallel + 1) - - if not index_step: - index_step = 1 - - result = [valid_revisions[index] \ - for index in range(0, len(valid_revisions), index_step)] - - return result - - -def GetRevisionsListAndHashList(start, end, parallel, src_path, - pending_revisions, skip_revisions): +def GetCommitsBetween(start, end, parallel, src_path, pending_revisions, + skip_revisions): """Determines the revisions between start and end.""" - new_llvm = get_llvm_hash.LLVMHash() + with get_llvm_hash.LLVMHash().CreateTempDirectory() as temp_dir: + # We have guaranteed contiguous revision numbers after this, + # and that guarnatee simplifies things considerably, so we don't + # support anything before it. + assert start >= git_llvm_rev.base_llvm_revision, f'{start} was too long ago' - with new_llvm.CreateTempDirectory() as temp_dir: with get_llvm_hash.CreateTempLLVMRepo(temp_dir) as new_repo: if not src_path: src_path = new_repo - - # Get a list of revisions between start and end. - revisions = GetRevisionsBetweenBisection( - start, end, parallel, src_path, pending_revisions, skip_revisions) - + index_step = (end - (start + 1)) // (parallel + 1) + if not index_step: + index_step = 1 + revisions = [ + rev for rev in range(start + 1, end, index_step) + if rev not in pending_revisions and rev not in skip_revisions + ] git_hashes = [ get_llvm_hash.GetGitHashFrom(src_path, rev) for rev in revisions ] + return revisions, git_hashes - return revisions, git_hashes - - -def DieWithNoRevisionsError(start, end, skip_revisions, pending_revisions): - """Raises a ValueError exception with useful information.""" - - no_revisions_message = ('No revisions between start %d and end ' - '%d to create tryjobs' % (start, end)) - if pending_revisions: - no_revisions_message += '\nThe following tryjobs are pending:\n' \ - + '\n'.join(str(rev) for rev in pending_revisions) - - if skip_revisions: - no_revisions_message += '\nThe following tryjobs were skipped:\n' \ - + '\n'.join(str(rev) for rev in skip_revisions) - - raise ValueError(no_revisions_message) - - -def CheckForExistingTryjobsInRevisionsToLaunch(revisions, jobs): - """Checks if a revision in 'revisions' exists in 'jobs' list.""" - - for rev in revisions: - if update_tryjob_status.FindTryjobIndex(rev, jobs) is not None: - raise ValueError('Revision %d exists already in "jobs"' % rev) - - -def UpdateBisection(revisions, git_hashes, bisect_contents, last_tested, - update_packages, chroot_path, patch_metadata_file, - extra_change_lists, options, builder, verbose): +def Bisect(revisions, git_hashes, bisect_state, last_tested, update_packages, + chroot_path, patch_metadata_file, extra_change_lists, options, + builder, verbose): """Adds tryjobs and updates the status file with the new tryjobs.""" try: for svn_revision, git_hash in zip(revisions, git_hashes): - tryjob_dict = modify_a_tryjob.AddTryjob( - update_packages, git_hash, svn_revision, chroot_path, - patch_metadata_file, extra_change_lists, options, builder, verbose, - svn_revision) + tryjob_dict = modify_a_tryjob.AddTryjob(update_packages, git_hash, + svn_revision, chroot_path, + patch_metadata_file, + extra_change_lists, options, + builder, verbose, svn_revision) - bisect_contents['jobs'].append(tryjob_dict) + bisect_state['jobs'].append(tryjob_dict) finally: # Do not want to lose progress if there is an exception. if last_tested: new_file = '%s.new' % last_tested with open(new_file, 'w') as json_file: - json.dump(bisect_contents, json_file, indent=4, separators=(',', ': ')) + json.dump(bisect_state, json_file, indent=4, separators=(',', ': ')) os.rename(new_file, last_tested) -def _NoteCompletedBisection(last_tested, src_path, end): - """Prints that bisection is complete.""" - - print('Finished bisecting for %s' % last_tested) - - if src_path: - bad_llvm_hash = get_llvm_hash.GetGitHashFrom(src_path, end) - else: - bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end) - - print( - 'The bad revision is %d and its commit hash is %s' % (end, bad_llvm_hash)) - - def LoadStatusFile(last_tested, start, end): """Loads the status file for bisection.""" @@ -383,37 +288,36 @@ def LoadStatusFile(last_tested, start, end): def main(args_output): - """Bisects LLVM based off of a .JSON file. + """Bisects LLVM commits. Raises: AssertionError: The script was run inside the chroot. """ chroot.VerifyOutsideChroot() - update_packages = [ 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' ] - patch_metadata_file = 'PATCHES.json' - start = args_output.start_rev end = args_output.end_rev - bisect_contents = LoadStatusFile(args_output.last_tested, start, end) - - _ValidateStartAndEndAgainstJSONStartAndEnd( - start, end, bisect_contents['start'], bisect_contents['end']) + bisect_state = LoadStatusFile(args_output.last_tested, start, end) + if start != bisect_state['start'] or end != bisect_state['end']: + raise ValueError(f'The start {start} or the end {end} version provided is ' + f'different than "start" {bisect_state["start"]} or "end" ' + f'{bisect_state["end"]} in the .JSON file') # Pending and skipped revisions are between 'start_revision' and # 'end_revision'. start_revision, end_revision, pending_revisions, skip_revisions = \ - GetStartAndEndRevision(start, end, bisect_contents['jobs']) + GetRemainingRange(start, end, bisect_state['jobs']) - revisions, git_hashes = GetRevisionsListAndHashList( - start_revision, end_revision, args_output.parallel, args_output.src_path, - pending_revisions, skip_revisions) + revisions, git_hashes = GetCommitsBetween(start_revision, end_revision, + args_output.parallel, + args_output.src_path, + pending_revisions, skip_revisions) # No more revisions between 'start_revision' and 'end_revision', so # bisection is complete. @@ -421,39 +325,61 @@ def main(args_output): # This is determined by finding all valid revisions between 'start_revision' # and 'end_revision' and that are NOT in the 'pending' and 'skipped' set. if not revisions: - # Successfully completed bisection where there are 2 cases: - # 1) 'start_revision' and 'end_revision' are back-to-back (example: - # 'start_revision' is 369410 and 'end_revision' is 369411). - # - # 2) 'start_revision' and 'end_revision' are NOT back-to-back, so there must - # be tryjobs in between which are labeled as 'skip' for their 'status' - # value. - # - # In either case, there are no 'pending' jobs. - if not pending_revisions: - _NoteCompletedBisection(args_output.last_tested, args_output.src_path, - end_revision) + if pending_revisions: + # Some tryjobs are not finished which may change the actual bad + # commit/revision when those tryjobs are finished. + no_revisions_message = (f'No revisions between start {start_revision} ' + f'and end {end_revision} to create tryjobs\n') - if skip_revisions: - skip_revisions_message = ('\nThe following revisions were skipped:\n' + - '\n'.join(str(rev) for rev in skip_revisions)) + if pending_revisions: + no_revisions_message += ( + 'The following tryjobs are pending:\n' + + '\n'.join(str(rev) for rev in pending_revisions) + '\n') - print(skip_revisions_message) - - return BisectionExitStatus.BISECTION_COMPLETE.value - - # Some tryjobs are not finished which may change the actual bad - # commit/revision when those tryjobs are finished. - DieWithNoRevisionsError(start_revision, end_revision, skip_revisions, - pending_revisions) - - CheckForExistingTryjobsInRevisionsToLaunch(revisions, bisect_contents['jobs']) + if skip_revisions: + no_revisions_message += ('The following tryjobs were skipped:\n' + + '\n'.join(str(rev) for rev in skip_revisions) + + '\n') + + raise ValueError(no_revisions_message) + + print(f'Finished bisecting for {args_output.last_tested}') + if args_output.src_path: + bad_llvm_hash = get_llvm_hash.GetGitHashFrom(args_output.src_path, + end_revision) + else: + bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_revision) + print(f'The bad revision is {end_revision} and its commit hash is ' + f'{bad_llvm_hash}') + if skip_revisions: + skip_revisions_message = ('\nThe following revisions were skipped:\n' + + '\n'.join(str(rev) for rev in skip_revisions)) + print(skip_revisions_message) + + if args_output.cleanup: + # Abondon all the CLs created for bisection + gerrit = os.path.join(args_output.chroot_path, 'chromite/bin/gerrit') + for build in bisect_state['jobs']: + try: + subprocess.check_output([gerrit, 'abandon', build['cl']], + stderr=subprocess.STDOUT, + encoding='utf-8') + except subprocess.CalledProcessError as err: + # the CL may have been abandoned + if 'chromite.lib.gob_util.GOBError' not in err.output: + raise + + return BisectionExitStatus.BISECTION_COMPLETE.value - UpdateBisection(revisions, git_hashes, bisect_contents, - args_output.last_tested, update_packages, - args_output.chroot_path, patch_metadata_file, - args_output.extra_change_lists, args_output.options, - args_output.builder, args_output.verbose) + for rev in revisions: + if update_tryjob_status.FindTryjobIndex(rev, + bisect_state['jobs']) is not None: + raise ValueError(f'Revision {rev} exists already in "jobs"') + + Bisect(revisions, git_hashes, bisect_state, args_output.last_tested, + update_packages, args_output.chroot_path, patch_metadata_file, + args_output.extra_change_lists, args_output.options, + args_output.builder, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/llvm_bisection_unittest.py b/llvm_tools/llvm_bisection_unittest.py index e730293b..a40770a5 100755 --- a/llvm_tools/llvm_bisection_unittest.py +++ b/llvm_tools/llvm_bisection_unittest.py @@ -11,50 +11,57 @@ from __future__ import print_function import json +import os +import subprocess import unittest import unittest.mock as mock import chroot import get_llvm_hash +import git_llvm_rev import llvm_bisection import modify_a_tryjob import test_helpers -import update_tryjob_status class LLVMBisectionTest(unittest.TestCase): """Unittests for LLVM bisection.""" - def testStartAndEndDoNotMatchJsonStartAndEnd(self): + def testGetRemainingRangePassed(self): start = 100 end = 150 - json_start = 110 - json_end = 150 - - # Verify the exception is raised when the start and end revision for LLVM - # bisection do not match the .JSON's 'start' and 'end' values. - with self.assertRaises(ValueError) as err: - llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd( - start, end, json_start, json_end) - - expected_error_message = ('The start %d or the end %d version provided is ' - 'different than "start" %d or "end" %d in the ' - '.JSON file' % (start, end, json_start, json_end)) - - self.assertEqual(str(err.exception), expected_error_message) - - def testStartAndEndMatchJsonStartAndEnd(self): - start = 100 - end = 150 + test_tryjobs = [{ + 'rev': 110, + 'status': 'good', + 'link': 'https://some_tryjob_1_url.com' + }, { + 'rev': 120, + 'status': 'good', + 'link': 'https://some_tryjob_2_url.com' + }, { + 'rev': 130, + 'status': 'pending', + 'link': 'https://some_tryjob_3_url.com' + }, { + 'rev': 135, + 'status': 'skip', + 'link': 'https://some_tryjob_4_url.com' + }, { + 'rev': 140, + 'status': 'bad', + 'link': 'https://some_tryjob_5_url.com' + }] - json_start = 100 - json_end = 150 + # Tuple consists of the new good revision, the new bad revision, a set of + # 'pending' revisions, and a set of 'skip' revisions. + expected_revisions_tuple = 120, 140, {130}, {135} - llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd( - start, end, json_start, json_end) + self.assertEqual( + llvm_bisection.GetRemainingRange(start, end, test_tryjobs), + expected_revisions_tuple) - def testTryjobStatusIsMissing(self): + def testGetRemainingRangeFailedWithMissingStatus(self): start = 100 end = 150 @@ -72,18 +79,14 @@ class LLVMBisectionTest(unittest.TestCase): 'link': 'https://some_tryjob_3_url.com' }] - # Verify the exception is raised when a tryjob does not have a value for - # the 'status' key or the 'status' key is missing. with self.assertRaises(ValueError) as err: - llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs) + llvm_bisection.GetRemainingRange(start, end, test_tryjobs) - expected_error_message = ( - '"status" is missing or has no value, please ' - 'go to %s and update it' % test_tryjobs[1]['link']) + error_message = ('"status" is missing or has no value, please ' + 'go to %s and update it' % test_tryjobs[1]['link']) + self.assertEqual(str(err.exception), error_message) - self.assertEqual(str(err.exception), expected_error_message) - - def testGoodRevisionGreaterThanBadRevision(self): + def testGetRemainingRangeFailedWithInvalidRange(self): start = 100 end = 150 @@ -101,206 +104,68 @@ class LLVMBisectionTest(unittest.TestCase): 'link': 'https://some_tryjob_3_url.com' }] - # Verify the exception is raised when the new 'start' revision is greater - # than the new 'bad' revision for bisection (i.e. bisection is broken). with self.assertRaises(AssertionError) as err: - llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs) + llvm_bisection.GetRemainingRange(start, end, test_tryjobs) - expected_error_message = ( - 'Bisection is broken because %d (good) is >= ' - '%d (bad)' % (test_tryjobs[2]['rev'], test_tryjobs[0]['rev'])) + expected_error_message = ('Bisection is broken because %d (good) is >= ' + '%d (bad)' % + (test_tryjobs[2]['rev'], test_tryjobs[0]['rev'])) self.assertEqual(str(err.exception), expected_error_message) - def testSuccessfullyGetNewStartAndNewEndRevision(self): - start = 100 - end = 150 - - test_tryjobs = [{ - 'rev': 110, - 'status': 'good', - 'link': 'https://some_tryjob_1_url.com' - }, { - 'rev': 120, - 'status': 'good', - 'link': 'https://some_tryjob_2_url.com' - }, { - 'rev': 130, - 'status': 'pending', - 'link': 'https://some_tryjob_3_url.com' - }, { - 'rev': 135, - 'status': 'skip', - 'link': 'https://some_tryjob_4_url.com' - }, { - 'rev': 140, - 'status': 'bad', - 'link': 'https://some_tryjob_5_url.com' - }] - - # Tuple consists of the new good revision, the new bad revision, a set of - # 'pending' revisions, and a set of 'skip' revisions. - expected_revisions_tuple = 120, 140, {130}, {135} - - self.assertTupleEqual( - llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs), - expected_revisions_tuple) - @mock.patch.object(get_llvm_hash, 'GetGitHashFrom') - def testNoRevisionsBetweenStartAndEnd(self, mock_get_git_hash): - start = 100 - end = 110 - - test_pending_revisions = {107} - test_skip_revisions = {101, 102, 103, 104, 108, 109} - - # Simulate behavior of `GetGitHashFrom()` when the revision does not - # exist in the LLVM source tree. - def MockGetGitHashForRevisionRaiseException(_src_path, _revision): - raise ValueError('Revision does not exist') - - mock_get_git_hash.side_effect = MockGetGitHashForRevisionRaiseException - - parallel = 3 - - abs_path_to_src = '/abs/path/to/src' - - self.assertListEqual( - llvm_bisection.GetRevisionsBetweenBisection( - start, end, parallel, abs_path_to_src, test_pending_revisions, - test_skip_revisions), []) - - # Assume llvm_bisection module has imported GetGitHashFrom - @mock.patch.object(get_llvm_hash, 'GetGitHashFrom') - def testSuccessfullyRetrievedRevisionsBetweenStartAndEnd( - self, mock_get_git_hash): - - start = 100 - end = 110 - - test_pending_revisions = set() - test_skip_revisions = {101, 102, 103, 104, 106, 108, 109} - + def testGetCommitsBetweenPassed(self, mock_get_git_hash): + start = git_llvm_rev.base_llvm_revision + end = start + 10 + test_pending_revisions = {start + 7} + test_skip_revisions = { + start + 1, start + 2, start + 4, start + 8, start + 9 + } parallel = 3 - abs_path_to_src = '/abs/path/to/src' - # Valid revision that exist in the LLVM source tree between 'start' and - # 'end' and were not in the 'pending' set or 'skip' set. - expected_revisions_between_start_and_end = [105, 107] - - self.assertListEqual( - llvm_bisection.GetRevisionsBetweenBisection( - start, end, parallel, abs_path_to_src, test_pending_revisions, - test_skip_revisions), expected_revisions_between_start_and_end) - - self.assertEqual(mock_get_git_hash.call_count, 2) + revs = ['a123testhash3', 'a123testhash5'] + mock_get_git_hash.side_effect = revs - # Simulate behavior of `GetGitHashFrom()` when successfully retrieved - # a list git hashes for each revision in the revisions list. - # Assume llvm_bisection module has imported GetGitHashFrom - @mock.patch.object(get_llvm_hash, 'GetGitHashFrom') - # Simulate behavior of `GetRevisionsBetweenBisection()` when successfully - # retrieved a list of valid revisions between 'start' and 'end'. - @mock.patch.object(llvm_bisection, 'GetRevisionsBetweenBisection') - # Simulate behavior of `CreatTempLLVMRepo()` when successfully created a - # worktree when a source path was not provided. - @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo') - def testSuccessfullyGetRevisionsListAndHashList( - self, mock_create_temp_llvm_repo, mock_get_revisions_between_bisection, - mock_get_git_hash): - - expected_revisions_and_hash_tuple = ([102, 105, 108], [ - 'a123testhash1', 'a123testhash2', 'a123testhash3' - ]) - - @test_helpers.CallCountsToMockFunctions - def MockGetGitHashForRevision(call_count, _src_path, _rev): - # Simulate retrieving the git hash for the revision. - if call_count < 3: - return expected_revisions_and_hash_tuple[1][call_count] - - assert False, 'Called `GetGitHashFrom()` more than expected.' - - temp_worktree = '/abs/path/to/tmpDir' - - mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = \ - temp_worktree - - # Simulate the valid revisions list. - mock_get_revisions_between_bisection.return_value = \ - expected_revisions_and_hash_tuple[0] - - # Simulate behavior of `GetGitHashFrom()` by using the testing - # function. - mock_get_git_hash.side_effect = MockGetGitHashForRevision + git_hashes = [ + git_llvm_rev.base_llvm_revision + 3, git_llvm_rev.base_llvm_revision + 5 + ] - start = 100 - end = 110 - parallel = 3 - src_path = None - pending_revisions = {103, 104} - skip_revisions = {101, 106, 107, 109} - - self.assertTupleEqual( - llvm_bisection.GetRevisionsListAndHashList( - start, end, parallel, src_path, pending_revisions, skip_revisions), - expected_revisions_and_hash_tuple) - - mock_get_revisions_between_bisection.assert_called_once() - - self.assertEqual(mock_get_git_hash.call_count, 3) + self.assertEqual( + llvm_bisection.GetCommitsBetween(start, end, parallel, abs_path_to_src, + test_pending_revisions, + test_skip_revisions), + (git_hashes, revs)) - def testSuccessfullyDieWithNoRevisionsError(self): + def testLoadStatusFilePassedWithExistingFile(self): start = 100 - end = 110 - - pending_revisions = {105, 108} - skip_revisions = {101, 102, 103, 104, 106, 107, 109} - - expected_no_revisions_message = ('No revisions between start %d and end ' - '%d to create tryjobs' % (start, end)) - - expected_no_revisions_message += '\nThe following tryjobs are pending:\n' \ - + '\n'.join(str(rev) for rev in pending_revisions) - - expected_no_revisions_message += '\nThe following tryjobs were skipped:\n' \ - + '\n'.join(str(rev) for rev in skip_revisions) - - # Verify that an exception is raised when there are no revisions to launch - # tryjobs for between 'start' and 'end' and some tryjobs are 'pending'. - with self.assertRaises(ValueError) as err: - llvm_bisection.DieWithNoRevisionsError(start, end, skip_revisions, - pending_revisions) + end = 150 - self.assertEqual(str(err.exception), expected_no_revisions_message) + test_bisect_state = {'start': start, 'end': end, 'jobs': []} - # Simulate behavior of `FindTryjobIndex()` when the index of the tryjob was - # found. - @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) - def testTryjobExistsInRevisionsToLaunch(self, mock_find_tryjob_index): - test_existing_jobs = [{'rev': 102, 'status': 'good'}] + # Simulate that the status file exists. + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + test_helpers.WritePrettyJsonFile(test_bisect_state, f) - revision_to_launch = [102] + self.assertEqual( + llvm_bisection.LoadStatusFile(temp_json_file, start, end), + test_bisect_state) - expected_revision_that_exists = 102 + def testLoadStatusFilePassedWithoutExistingFile(self): + start = 200 + end = 250 - with self.assertRaises(ValueError) as err: - llvm_bisection.CheckForExistingTryjobsInRevisionsToLaunch( - revision_to_launch, test_existing_jobs) + expected_bisect_state = {'start': start, 'end': end, 'jobs': []} - expected_found_tryjob_index_error_message = ( - 'Revision %d exists already ' - 'in "jobs"' % expected_revision_that_exists) + last_tested = '/abs/path/to/file_that_does_not_exist.json' self.assertEqual( - str(err.exception), expected_found_tryjob_index_error_message) - - mock_find_tryjob_index.assert_called_once() + llvm_bisection.LoadStatusFile(last_tested, start, end), + expected_bisect_state) @mock.patch.object(modify_a_tryjob, 'AddTryjob') - def testSuccessfullyUpdatedStatusFileWhenExceptionIsRaised( - self, mock_add_tryjob): + def testBisectPassed(self, mock_add_tryjob): git_hash_list = ['a123testhash1', 'a123testhash2', 'a123testhash3'] revisions_list = [102, 104, 106] @@ -343,11 +208,11 @@ class LLVMBisectionTest(unittest.TestCase): # Verify that the status file is updated when an exception happened when # attempting to launch a revision (i.e. progress is not lost). with self.assertRaises(ValueError) as err: - llvm_bisection.UpdateBisection( - revisions_list, git_hash_list, bisection_contents, temp_json_file, - packages, args_output.chroot_path, patch_file, - args_output.extra_change_lists, args_output.options, - args_output.builders, args_output.verbose) + llvm_bisection.Bisect(revisions_list, git_hash_list, bisection_contents, + temp_json_file, packages, args_output.chroot_path, + patch_file, args_output.extra_change_lists, + args_output.options, args_output.builders, + args_output.verbose) expected_bisection_contents = { 'start': @@ -368,121 +233,143 @@ class LLVMBisectionTest(unittest.TestCase): with open(temp_json_file) as f: json_contents = json.load(f) - self.assertDictEqual(json_contents, expected_bisection_contents) + self.assertEqual(json_contents, expected_bisection_contents) self.assertEqual(str(err.exception), 'Unable to launch tryjob') self.assertEqual(mock_add_tryjob.call_count, 3) - # Simulate behavior of `GetGitHashFrom()` when successfully retrieved - # the git hash of the bad revision. Assume llvm_bisection has imported - # GetGitHashFrom + @mock.patch.object(subprocess, 'check_output', return_value=None) @mock.patch.object( - get_llvm_hash, 'GetGitHashFrom', return_value='a123testhash4') - def testCompletedBisectionWhenProvidedSrcPath(self, mock_get_git_hash): - last_tested = '/some/last/tested_file.json' + get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') + @mock.patch.object(llvm_bisection, 'GetCommitsBetween') + @mock.patch.object(llvm_bisection, 'GetRemainingRange') + @mock.patch.object(llvm_bisection, 'LoadStatusFile') + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) + def testMainPassed(self, mock_outside_chroot, mock_load_status_file, + mock_get_range, mock_get_revision_and_hash_list, + _mock_get_bad_llvm_hash, mock_abandon_cl): - src_path = '/abs/path/to/src/path' + start = 500 + end = 502 + cl = 1 - # The bad revision. - end = 150 + bisect_state = { + 'start': start, + 'end': end, + 'jobs': [{ + 'rev': 501, + 'status': 'bad', + 'cl': cl + }] + } - llvm_bisection._NoteCompletedBisection(last_tested, src_path, end) + skip_revisions = {501} + pending_revisions = {} - mock_get_git_hash.assert_called_once() + mock_load_status_file.return_value = bisect_state - # Simulate behavior of `GetLLVMHash()` when successfully retrieved - # the git hash of the bad revision. - @mock.patch.object( - get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash5') - def testCompletedBisectionWhenNotProvidedSrcPath(self, mock_get_git_hash): - last_tested = '/some/last/tested_file.json' + mock_get_range.return_value = (start, end, pending_revisions, + skip_revisions) - src_path = None + mock_get_revision_and_hash_list.return_value = [], [] - # The bad revision. - end = 200 + args_output = test_helpers.ArgsOutputTest() + args_output.start_rev = start + args_output.end_rev = end + args_output.parallel = 3 + args_output.src_path = None + args_output.chroot_path = 'somepath' + args_output.cleanup = True - llvm_bisection._NoteCompletedBisection(last_tested, src_path, end) + self.assertEqual( + llvm_bisection.main(args_output), + llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value) - mock_get_git_hash.assert_called_once() + mock_outside_chroot.assert_called_once() - def testSuccessfullyLoadedStatusFile(self): - start = 100 - end = 150 + mock_load_status_file.assert_called_once() - test_bisect_contents = {'start': start, 'end': end, 'jobs': []} + mock_get_range.assert_called_once() - # Simulate that the status file exists. - with test_helpers.CreateTemporaryJsonFile() as temp_json_file: - with open(temp_json_file, 'w') as f: - test_helpers.WritePrettyJsonFile(test_bisect_contents, f) + mock_get_revision_and_hash_list.assert_called_once() - self.assertDictEqual( - llvm_bisection.LoadStatusFile(temp_json_file, start, end), - test_bisect_contents) + mock_abandon_cl.assert_called_once() + self.assertEqual( + mock_abandon_cl.call_args, + mock.call( + [ + os.path.join(args_output.chroot_path, 'chromite/bin/gerrit'), + 'abandon', + cl, + ], + stderr=subprocess.STDOUT, + encoding='utf-8', + )) - def testLoadedStatusFileThatDoesNotExist(self): - start = 200 - end = 250 + @mock.patch.object(llvm_bisection, 'LoadStatusFile') + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) + def testMainFailedWithInvalidRange(self, mock_outside_chroot, + mock_load_status_file): - expected_bisect_contents = {'start': start, 'end': end, 'jobs': []} + start = 500 + end = 502 - last_tested = '/abs/path/to/file_that_does_not_exist.json' + bisect_state = { + 'start': start - 1, + 'end': end, + } - self.assertDictEqual( - llvm_bisection.LoadStatusFile(last_tested, start, end), - expected_bisect_contents) - - # Simulate behavior of `_NoteCompletedBisection()` when there are no more - # tryjobs to launch between start and end, so bisection is complete. - @mock.patch.object(llvm_bisection, '_NoteCompletedBisection') - @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList') - @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision') - # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when - # both start and end revisions match. - @mock.patch.object(llvm_bisection, - '_ValidateStartAndEndAgainstJSONStartAndEnd') + mock_load_status_file.return_value = bisect_state + + args_output = test_helpers.ArgsOutputTest() + args_output.start_rev = start + args_output.end_rev = end + args_output.parallel = 3 + args_output.src_path = None + + with self.assertRaises(ValueError) as err: + llvm_bisection.main(args_output) + + error_message = (f'The start {start} or the end {end} version provided is ' + f'different than "start" {bisect_state["start"]} or "end" ' + f'{bisect_state["end"]} in the .JSON file') + + self.assertEqual(str(err.exception), error_message) + + mock_outside_chroot.assert_called_once() + + mock_load_status_file.assert_called_once() + + @mock.patch.object(llvm_bisection, 'GetCommitsBetween') + @mock.patch.object(llvm_bisection, 'GetRemainingRange') @mock.patch.object(llvm_bisection, 'LoadStatusFile') - # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the - # script outside of the chroot. @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) - def testSuccessfullyBisectedLLVM( - self, mock_outside_chroot, mock_load_status_file, - mock_validate_start_and_end, mock_get_start_and_end_revision, - mock_get_revision_and_hash_list, mock_note_completed_bisection): + def testMainFailedWithPendingBuilds(self, mock_outside_chroot, + mock_load_status_file, mock_get_range, + mock_get_revision_and_hash_list): start = 500 end = 502 + rev = 501 - bisect_contents = { + bisect_state = { 'start': start, 'end': end, 'jobs': [{ - 'rev': 501, - 'status': 'skip' + 'rev': rev, + 'status': 'pending' }] } - skip_revisions = {501} - pending_revisions = {} + skip_revisions = {} + pending_revisions = {rev} + + mock_load_status_file.return_value = bisect_state + + mock_get_range.return_value = (start, end, pending_revisions, + skip_revisions) - # Simulate behavior of `LoadStatusFile()` when successfully loaded the - # status file. - mock_load_status_file.return_value = bisect_contents - - # Simulate behavior of `GetStartAndEndRevision()` when successfully found - # the new start and end revision of the bisection. - # - # Returns new start revision, new end revision, a set of pending revisions, - # and a set of skip revisions. - mock_get_start_and_end_revision.return_value = (start, end, - pending_revisions, - skip_revisions) - - # Simulate behavior of `GetRevisionsListAndHashList()` when successfully - # retrieved valid revisions (along with their git hashes) between start and - # end (in this case, none). mock_get_revision_and_hash_list.return_value = [], [] args_output = test_helpers.ArgsOutputTest() @@ -491,110 +378,131 @@ class LLVMBisectionTest(unittest.TestCase): args_output.parallel = 3 args_output.src_path = None - self.assertEqual( - llvm_bisection.main(args_output), - llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value) + with self.assertRaises(ValueError) as err: + llvm_bisection.main(args_output) + + error_message = (f'No revisions between start {start} and end {end} to ' + 'create tryjobs\nThe following tryjobs are pending:\n' + f'{rev}\n') + + self.assertEqual(str(err.exception), error_message) mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() - mock_validate_start_and_end.assert_called_once() - - mock_get_start_and_end_revision.assert_called_once() + mock_get_range.assert_called_once() mock_get_revision_and_hash_list.assert_called_once() - mock_note_completed_bisection.assert_called_once() - - @mock.patch.object(llvm_bisection, 'DieWithNoRevisionsError') - # Simulate behavior of `_NoteCompletedBisection()` when there are no more - # tryjobs to launch between start and end, so bisection is complete. - @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList') - @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision') - # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when - # both start and end revisions match. - @mock.patch.object(llvm_bisection, - '_ValidateStartAndEndAgainstJSONStartAndEnd') + @mock.patch.object(llvm_bisection, 'GetCommitsBetween') + @mock.patch.object(llvm_bisection, 'GetRemainingRange') @mock.patch.object(llvm_bisection, 'LoadStatusFile') - # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the - # script outside of the chroot. @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) - def testNoMoreTryjobsToLaunch( - self, mock_outside_chroot, mock_load_status_file, - mock_validate_start_and_end, mock_get_start_and_end_revision, - mock_get_revision_and_hash_list, mock_die_with_no_revisions_error): + def testMainFailedWithDuplicateBuilds(self, mock_outside_chroot, + mock_load_status_file, mock_get_range, + mock_get_revision_and_hash_list): start = 500 end = 502 + rev = 501 + git_hash = 'a123testhash1' - bisect_contents = { + bisect_state = { 'start': start, 'end': end, 'jobs': [{ - 'rev': 501, + 'rev': rev, 'status': 'pending' }] } skip_revisions = {} - pending_revisions = {501} - - no_revisions_error_message = ('No more tryjobs to launch between %d and ' - '%d' % (start, end)) - - def MockNoRevisionsErrorException(_start, _end, _skip, _pending): - raise ValueError(no_revisions_error_message) - - # Simulate behavior of `LoadStatusFile()` when successfully loaded the - # status file. - mock_load_status_file.return_value = bisect_contents - - # Simulate behavior of `GetStartAndEndRevision()` when successfully found - # the new start and end revision of the bisection. - # - # Returns new start revision, new end revision, a set of pending revisions, - # and a set of skip revisions. - mock_get_start_and_end_revision.return_value = (start, end, - pending_revisions, - skip_revisions) - - # Simulate behavior of `GetRevisionsListAndHashList()` when successfully - # retrieved valid revisions (along with their git hashes) between start and - # end (in this case, none). - mock_get_revision_and_hash_list.return_value = [], [] + pending_revisions = {rev} + + mock_load_status_file.return_value = bisect_state - # Use the test function to simulate `DieWithNoRevisionsWithError()` - # behavior. - mock_die_with_no_revisions_error.side_effect = MockNoRevisionsErrorException + mock_get_range.return_value = (start, end, pending_revisions, + skip_revisions) + + mock_get_revision_and_hash_list.return_value = [rev], [git_hash] - # Simulate behavior of arguments passed into the command line. args_output = test_helpers.ArgsOutputTest() args_output.start_rev = start args_output.end_rev = end args_output.parallel = 3 args_output.src_path = None - # Verify the exception is raised when there are no more tryjobs to launch - # between start and end when there are tryjobs that are 'pending', so - # the actual bad revision can change when those tryjobs's 'status' are - # updated. with self.assertRaises(ValueError) as err: llvm_bisection.main(args_output) - self.assertEqual(str(err.exception), no_revisions_error_message) + error_message = ('Revision %d exists already in "jobs"' % rev) + self.assertEqual(str(err.exception), error_message) mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() - mock_validate_start_and_end.assert_called_once() - - mock_get_start_and_end_revision.assert_called_once() + mock_get_range.assert_called_once() mock_get_revision_and_hash_list.assert_called_once() - mock_die_with_no_revisions_error.assert_called_once() + @mock.patch.object(subprocess, 'check_output', return_value=None) + @mock.patch.object( + get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') + @mock.patch.object(llvm_bisection, 'GetCommitsBetween') + @mock.patch.object(llvm_bisection, 'GetRemainingRange') + @mock.patch.object(llvm_bisection, 'LoadStatusFile') + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) + def testMainFailedToAbandonCL(self, mock_outside_chroot, + mock_load_status_file, mock_get_range, + mock_get_revision_and_hash_list, + _mock_get_bad_llvm_hash, mock_abandon_cl): + + start = 500 + end = 502 + + bisect_state = { + 'start': start, + 'end': end, + 'jobs': [{ + 'rev': 501, + 'status': 'bad', + 'cl': 0 + }] + } + + skip_revisions = {501} + pending_revisions = {} + + mock_load_status_file.return_value = bisect_state + + mock_get_range.return_value = (start, end, pending_revisions, + skip_revisions) + + mock_get_revision_and_hash_list.return_value = ([], []) + + error_message = 'Error message.' + mock_abandon_cl.side_effect = subprocess.CalledProcessError( + returncode=1, cmd=[], output=error_message) + + args_output = test_helpers.ArgsOutputTest() + args_output.start_rev = start + args_output.end_rev = end + args_output.parallel = 3 + args_output.src_path = None + args_output.cleanup = True + + with self.assertRaises(subprocess.CalledProcessError) as err: + llvm_bisection.main(args_output) + + self.assertEqual(err.exception.output, error_message) + + mock_outside_chroot.assert_called_once() + + mock_load_status_file.assert_called_once() + + mock_get_range.assert_called_once() if __name__ == '__main__': diff --git a/llvm_tools/llvm_project.py b/llvm_tools/llvm_project.py index c171370c..7937729f 100644 --- a/llvm_tools/llvm_project.py +++ b/llvm_tools/llvm_project.py @@ -13,6 +13,7 @@ import subprocess import sys import get_llvm_hash +import git_llvm_rev def get_location() -> str: @@ -33,9 +34,13 @@ def ensure_up_to_date(): assert checkout == actual_checkout, '%s != %s' % (actual_checkout, checkout) commit_timestamp = subprocess.check_output( - ['git', 'log', '-n1', '--format=%ct', 'origin/master'], + [ + 'git', 'log', '-n1', '--format=%ct', + 'origin/' + git_llvm_rev.MAIN_BRANCH + ], cwd=checkout, - encoding='utf-8') + encoding='utf-8', + ) commit_time = datetime.datetime.fromtimestamp(int(commit_timestamp.strip())) now = datetime.datetime.now() @@ -52,7 +57,7 @@ def ensure_up_to_date(): (time_since_last_commit.days, checkout), file=sys.stderr) - result = subprocess.run(['git', 'fetch', 'origin'], cwd=checkout) + result = subprocess.run(['git', 'fetch', 'origin'], check=False, cwd=checkout) if result.returncode: print( 'Sync failed somehow; hoping that things are fresh enough, then...', diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py index 71b5574b..3a23890a 100755 --- a/llvm_tools/nightly_revert_checker.py +++ b/llvm_tools/nightly_revert_checker.py @@ -27,6 +27,7 @@ import typing as t import cros_utils.email_sender as email_sender import cros_utils.tiny_render as tiny_render import get_llvm_hash +import git_llvm_rev import revert_checker State = t.Any @@ -44,12 +45,12 @@ def _find_interesting_android_shas( encoding='utf-8', ).strip() - master_legacy = get_llvm_merge_base('aosp/master-legacy') + main_legacy = get_llvm_merge_base('aosp/master-legacy') testing_upstream = get_llvm_merge_base('aosp/testing-upstream') - result = [('master-legacy', master_legacy)] + result = [('main-legacy', main_legacy)] # If these are the same SHA, there's no point in tracking both. - if master_legacy != testing_upstream: + if main_legacy != testing_upstream: result.append(('testing-upstream', testing_upstream)) return result @@ -270,7 +271,7 @@ def main(argv: t.List[str]) -> None: for friendly_name, sha in interesting_shas: logging.info('Finding reverts across %s (%s)', friendly_name, sha) all_reverts = revert_checker.find_reverts( - llvm_dir, sha, root='origin/master') + llvm_dir, sha, root='origin/' + git_llvm_rev.MAIN_BRANCH) logging.info('Detected the following revert(s) across %s:\n%s', friendly_name, pprint.pformat(all_reverts)) diff --git a/llvm_tools/upload_lexan_crashes_to_forcey.py b/llvm_tools/upload_lexan_crashes_to_forcey.py new file mode 100755 index 00000000..b93f51a7 --- /dev/null +++ b/llvm_tools/upload_lexan_crashes_to_forcey.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Fetches and submits the latest test-cases from Lexan's crash bucket.""" + +# pylint: disable=cros-logging-import + +import argparse +import contextlib +import datetime +import json +import logging +import os +import shutil +import subprocess +import sys +import tempfile +from typing import Generator, List, Iterable + +gsurl_base = 'gs://chrome-clang-crash-reports/v1' + + +def gsutil_ls(loc: str) -> List[str]: + results = subprocess.run(['gsutil.py', 'ls', loc], + stdout=subprocess.PIPE, + check=True, + encoding='utf-8') + return [l.strip() for l in results.stdout.splitlines()] + + +def gsurl_ls_last_numbers(url: str) -> List[int]: + return sorted(int(x.rstrip('/').split('/')[-1]) for x in gsutil_ls(url)) + + +def get_available_year_numbers() -> List[int]: + return gsurl_ls_last_numbers(gsurl_base) + + +def get_available_month_numbers(year: int) -> List[int]: + return gsurl_ls_last_numbers(f'{gsurl_base}/{year}') + + +def get_available_day_numbers(year: int, month: int) -> List[int]: + return gsurl_ls_last_numbers(f'{gsurl_base}/{year}/{month:02d}') + + +def get_available_test_case_urls(year: int, month: int, day: int) -> List[str]: + return gsutil_ls(f'{gsurl_base}/{year}/{month:02d}/{day:02d}') + + +def test_cases_on_or_after(date: datetime.datetime + ) -> Generator[str, None, None]: + """Yields all test-cases submitted on or after the given date.""" + for year in get_available_year_numbers(): + if year < date.year: + continue + + for month in get_available_month_numbers(year): + if year == date.year and month < date.month: + continue + + for day in get_available_day_numbers(year, month): + when = datetime.date(year, month, day) + if when < date: + continue + + yield when, get_available_test_case_urls(year, month, day) + + +def to_ymd(date: datetime.date) -> str: + return date.strftime('%Y-%m-%d') + + +def from_ymd(date_str: str) -> datetime.date: + return datetime.datetime.strptime(date_str, '%Y-%m-%d').date() + + +def persist_state(seen_urls: Iterable[str], state_file: str, + current_date: datetime.date): + tmp_state_file = state_file + '.tmp' + with open(tmp_state_file, 'w', encoding='utf-8') as f: + json.dump( + { + 'already_seen': sorted(seen_urls), + 'most_recent_date': to_ymd(current_date), + }, + f, + ) + os.rename(tmp_state_file, state_file) + + +@contextlib.contextmanager +def temp_dir() -> Generator[str, None, None]: + loc = tempfile.mkdtemp('lexan-autosubmit') + try: + yield loc + finally: + shutil.rmtree(loc) + + +def submit_test_case(gs_url: str, cr_tool: str) -> None: + logging.info('Submitting %s', gs_url) + suffix = os.path.splitext(gs_url)[1] + with temp_dir() as tempdir: + target_name = 'test_case' + suffix + target = os.path.join(tempdir, target_name) + subprocess.run(['gsutil.py', 'cp', gs_url, target], check=True) + subprocess.run(['tar', 'xaf', target_name], check=True, cwd=tempdir) + os.unlink(target) + + # Sometimes (e.g., in + # gs://chrome-clang-crash-reports/v1/2020/03/27/ + # chromium.clang-ToTiOS-12754-GTXToolKit-2bfcde.tgz) + # we'll get `.crash` files. Unclear why, but let's filter them out anyway. + repro_files = [x for x in os.listdir(tempdir) if not x.endswith('.crash')] + assert len(repro_files) == 2, repro_files + if repro_files[0].endswith('.sh'): + sh_file, src_file = repro_files + assert not src_file.endswith('.sh'), repro_files + else: + src_file, sh_file = repro_files + assert sh_file.endswith('.sh'), repro_files + + subprocess.run( + [ + cr_tool, + 'reduce', + '-stream=false', + '-wait=false', + '-note', + gs_url, + '-sh_file', + os.path.join(tempdir, sh_file), + '-src_file', + os.path.join(tempdir, src_file), + ], + check=True, + ) + + +def submit_new_test_cases( + last_seen_test_cases: Iterable[str], + earliest_date_to_check: datetime.date, + forcey: str, + state_file_path: str, +) -> None: + """Submits new test-cases to forcey. + + This will persist state after each test-case is submitted. + + Args: + last_seen_test_cases: test-cases which have been submitted already, and + should be skipped if seen again. + earliest_date_to_check: the earliest date we should consider test-cases + from. + forcey: path to the forcey binary. + state_file_path: path to our state file. + """ + # `all_test_cases_seen` is the union of all test-cases seen on this and prior + # invocations. It guarantees, in all cases we care about, that we won't + # submit the same test-case twice. `test_cases_seen_this_invocation` is + # persisted as "all of the test-cases we've seen on this and prior + # invocations" if we successfully submit _all_ test-cases. + # + # Since you can visualize the test-cases this script considers as a sliding + # window that only moves forward, if we saw a test-case on a prior iteration + # but no longer see it, we'll never see it again (since it fell out of our + # sliding window by being too old). Hence, keeping it around is + # pointless. + # + # We only persist this minimized set of test-cases if _everything_ succeeds, + # since if something fails below, there's a chance that we haven't revisited + # test-cases that we've already seen. + all_test_cases_seen = set(last_seen_test_cases) + test_cases_seen_this_invocation = [] + most_recent_date = earliest_date_to_check + for date, candidates in test_cases_on_or_after(earliest_date_to_check): + most_recent_date = max(most_recent_date, date) + + for url in candidates: + test_cases_seen_this_invocation.append(url) + if url in all_test_cases_seen: + continue + + all_test_cases_seen.add(url) + submit_test_case(url, forcey) + + # Persisting on each iteration of this loop isn't free, but it's the + # easiest way to not resubmit test-cases, and it's good to keep in mind + # that: + # - the state file will be small (<12KB, since it only keeps a few days + # worth of test-cases after the first run) + # - in addition to this, we're downloading+unzipping+reuploading multiple + # MB of test-case bytes. + # + # So comparatively, the overhead here probably isn't an issue. + persist_state(all_test_cases_seen, state_file_path, most_recent_date) + + persist_state(test_cases_seen_this_invocation, state_file_path, + most_recent_date) + + +def main(argv: List[str]): + logging.basicConfig( + format='>> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: ' + '%(message)s', + level=logging.INFO, + ) + + my_dir = os.path.dirname(os.path.abspath(__file__)) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--state_file', default=os.path.join(my_dir, 'lexan-state.json')) + parser.add_argument( + '--last_date', + help='The earliest date that we care about. All test cases from here ' + 'on will be picked up. Format is YYYY-MM-DD.') + parser.add_argument( + '--4c', dest='forcey', required=True, help='Path to a 4c client binary') + opts = parser.parse_args(argv) + + forcey = opts.forcey + state_file = opts.state_file + last_date_str = opts.last_date + + os.makedirs(os.path.dirname(state_file), 0o755, exist_ok=True) + + if last_date_str is None: + with open(state_file, encoding='utf-8') as f: + data = json.load(f) + most_recent_date = from_ymd(data['most_recent_date']) + submit_new_test_cases( + last_seen_test_cases=data['already_seen'], + # Note that we always subtract one day from this to avoid a race: + # uploads may appear slightly out-of-order (or builders may lag, or + # ...), so the last test-case uploaded for 2020/01/01 might appear + # _after_ the first test-case for 2020/01/02. Assuming that builders + # won't lag behind for over a day, the easiest way to handle this is to + # always check the previous and current days. + earliest_date_to_check=most_recent_date - datetime.timedelta(days=1), + forcey=forcey, + state_file_path=state_file, + ) + else: + submit_new_test_cases( + last_seen_test_cases=(), + earliest_date_to_check=from_ymd(last_date_str), + forcey=forcey, + state_file_path=state_file, + ) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/llvm_tools/upload_lexan_crashes_to_forcey_test.py b/llvm_tools/upload_lexan_crashes_to_forcey_test.py new file mode 100755 index 00000000..3c9c0d4b --- /dev/null +++ b/llvm_tools/upload_lexan_crashes_to_forcey_test.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for upload_lexan_crashes_to_forcey.""" + +import datetime +import unittest +import unittest.mock + +import upload_lexan_crashes_to_forcey + + +class Test(unittest.TestCase): + """Tests for upload_lexan_crashes_to_forcey.""" + + def test_date_parsing_functions(self): + self.assertEqual( + datetime.date(2020, 2, 1), + upload_lexan_crashes_to_forcey.from_ymd('2020-02-01')) + + @unittest.mock.patch( + 'upload_lexan_crashes_to_forcey.test_cases_on_or_after', + return_value=( + ( + datetime.date(2020, 1, 1), + ('gs://test-case-1', 'gs://test-case-1.1'), + ), + (datetime.date(2020, 1, 2), ('gs://test-case-2',)), + (datetime.date(2020, 1, 1), ('gs://test-case-3',)), + (datetime.date(2020, 1, 4), ('gs://test-case-4',)), + )) + @unittest.mock.patch('upload_lexan_crashes_to_forcey.submit_test_case') + @unittest.mock.patch('upload_lexan_crashes_to_forcey.persist_state') + def test_new_test_case_submission_functions(self, persist_state_mock, + submit_test_case_mock, + test_cases_on_or_after_mock): + forcey_path = '/path/to/4c' + real_state_file_path = '/path/to/state/file' + earliest_date = datetime.date(2020, 1, 1) + + persist_state_calls = [] + + # Since the set this gets is mutated, we need to copy it somehow. + def persist_state_side_effect(test_cases_to_persist, state_file_path, + most_recent_date): + self.assertEqual(state_file_path, real_state_file_path) + persist_state_calls.append( + (sorted(test_cases_to_persist), most_recent_date)) + + persist_state_mock.side_effect = persist_state_side_effect + + upload_lexan_crashes_to_forcey.submit_new_test_cases( + last_seen_test_cases=( + 'gs://test-case-0', + 'gs://test-case-1', + ), + earliest_date_to_check=earliest_date, + forcey=forcey_path, + state_file_path=real_state_file_path, + ) + + test_cases_on_or_after_mock.assert_called_once_with(earliest_date) + self.assertEqual(submit_test_case_mock.call_args_list, [ + unittest.mock.call('gs://test-case-1.1', forcey_path), + unittest.mock.call('gs://test-case-2', forcey_path), + unittest.mock.call('gs://test-case-3', forcey_path), + unittest.mock.call('gs://test-case-4', forcey_path), + ]) + + self.assertEqual(persist_state_calls, [ + ( + ['gs://test-case-0', 'gs://test-case-1', 'gs://test-case-1.1'], + datetime.date(2020, 1, 1), + ), + ( + [ + 'gs://test-case-0', + 'gs://test-case-1', + 'gs://test-case-1.1', + 'gs://test-case-2', + ], + datetime.date(2020, 1, 2), + ), + ( + [ + 'gs://test-case-0', + 'gs://test-case-1', + 'gs://test-case-1.1', + 'gs://test-case-2', + 'gs://test-case-3', + ], + datetime.date(2020, 1, 2), + ), + ( + [ + 'gs://test-case-0', + 'gs://test-case-1', + 'gs://test-case-1.1', + 'gs://test-case-2', + 'gs://test-case-3', + 'gs://test-case-4', + ], + datetime.date(2020, 1, 4), + ), + ( + [ + 'gs://test-case-1', + 'gs://test-case-1.1', + 'gs://test-case-2', + 'gs://test-case-3', + 'gs://test-case-4', + ], + datetime.date(2020, 1, 4), + ), + ]) + + +if __name__ == '__main__': + unittest.main() diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py new file mode 100755 index 00000000..cb41eb88 --- /dev/null +++ b/pgo_tools/monitor_pgo_profiles.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Emails the mage if PGO profile generation hasn't succeeded recently.""" + +# pylint: disable=cros-logging-import + +import argparse +import datetime +import sys +import subprocess +import logging +from typing import List, NamedTuple, Optional, Tuple + +from cros_utils import email_sender +from cros_utils import tiny_render + +PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/' + 'pgo-generate-llvm-next-orchestrator') + + +class ProfdataInfo(NamedTuple): + """Data about an llvm profdata in our gs:// bucket.""" + date: datetime.datetime + location: str + + +def parse_date(date: str) -> datetime.datetime: + time_format = '%Y-%m-%dT%H:%M:%SZ' + if not date.endswith('Z'): + time_format += '%z' + return datetime.datetime.strptime(date, time_format) + + +def fetch_most_recent_profdata(arch: str) -> ProfdataInfo: + result = subprocess.run( + [ + 'gsutil.py', + 'ls', + '-l', + f'gs://chromeos-toolchain-artifacts/llvm-pgo/{arch}/' + '*.profdata.tar.xz', + ], + check=True, + stdout=subprocess.PIPE, + encoding='utf-8', + ) + + # Each line will be a profdata; the last one is a summary, so drop it. + infos = [] + for rec in result.stdout.strip().splitlines()[:-1]: + _size, date, url = rec.strip().split() + infos.append(ProfdataInfo(date=parse_date(date), location=url)) + return max(infos) + + +def compose_complaint_email( + out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]] +) -> Optional[Tuple[str, tiny_render.Piece]]: + if not out_of_date_profiles: + return None + + if len(out_of_date_profiles) == 1: + subject = '1 llvm profile is out of date' + body = ['out-of-date profile:'] + else: + subject = f'{len(out_of_date_profiles)} llvm profiles are out of date' + body = ['out-of-date profiles:'] + + out_of_date_items = [] + for arch, profdata_info in out_of_date_profiles: + out_of_date_items.append( + f'{arch} (most recent profile was from {profdata_info.date} at ' + f'{profdata_info.location!r})') + + body += [ + tiny_render.UnorderedList(out_of_date_items), + tiny_render.line_break, + tiny_render.line_break, + 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' + 'Their status can be found at ', + tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK), + '.', + ] + return subject, body + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--dry_run', + action='store_true', + help="Don't actually send an email", + ) + parser.add_argument( + '--max_age_days', + # These builders run ~weekly. If we fail to generate two in a row, + # something's probably wrong. + default=15, + type=int, + help='How old to let profiles get before complaining, in days', + ) + args = parser.parse_args() + + now = datetime.datetime.now() + logging.info('Start time is %r', now) + + max_age = datetime.timedelta(days=args.max_age_days) + out_of_date_profiles = [] + for arch in ('arm', 'arm64', 'amd64'): + logging.info('Fetching most recent profdata for %r', arch) + most_recent = fetch_most_recent_profdata(arch) + logging.info('Most recent profdata for %r is %r', arch, most_recent) + + age = now - most_recent.date + if age >= max_age: + out_of_date_profiles.append((arch, most_recent)) + + email = compose_complaint_email(out_of_date_profiles) + if not email: + logging.info('No email to send; quit') + return + + subject, body = email + + identifier = 'llvm-pgo-monitor' + subject = f'[{identifier}] {subject}' + + logging.info('Sending email with title %r', subject) + if args.dry_run: + logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject, + tiny_render.render_text_pieces(body)) + else: + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier=identifier, + well_known_recipients=['mage'], + direct_recipients=['gbiv@google.com'], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pgo_tools/monitor_pgo_profiles_unittest.py b/pgo_tools/monitor_pgo_profiles_unittest.py new file mode 100755 index 00000000..b4e085ec --- /dev/null +++ b/pgo_tools/monitor_pgo_profiles_unittest.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for monitor_pgo_profiles.""" + +import datetime +import subprocess +import unittest +import unittest.mock + +import monitor_pgo_profiles +from cros_utils import tiny_render + + +class Test(unittest.TestCase): + """Tests for monitor_pgo_profiles.""" + + def test_compose_complaint_email_with_zero_out_of_date(self): + self.assertIsNone(monitor_pgo_profiles.compose_complaint_email([])) + + def test_compose_complaint_email_with_one_out_of_date(self): + profdata_info = monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 1, 2, 3, 4, 5), + location='gs://somewhere', + ) + result = monitor_pgo_profiles.compose_complaint_email([ + ('some_arch', profdata_info), + ]) + self.assertEqual(result, ('1 llvm profile is out of date', [ + 'out-of-date profile:', + tiny_render.UnorderedList([ + f'some_arch (most recent profile was from {profdata_info.date} at ' + f'{profdata_info.location!r})' + ]), + tiny_render.line_break, + tiny_render.line_break, + 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' + 'Their status can be found at ', + tiny_render.Link( + href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + ), + '.', + ])) + + def test_compose_complaint_email_with_two_out_of_date(self): + profdata_info_1 = monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 1, 2, 3, 4, 5), + location='gs://somewhere', + ) + profdata_info_2 = monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 3, 2, 1, 4, 5), + location='gs://somewhere-else', + ) + result = monitor_pgo_profiles.compose_complaint_email([ + ('some_arch', profdata_info_1), + ('some_other_arch', profdata_info_2), + ]) + self.assertEqual(result, ('2 llvm profiles are out of date', [ + 'out-of-date profiles:', + tiny_render.UnorderedList([ + f'some_arch (most recent profile was from {profdata_info_1.date} ' + f'at {profdata_info_1.location!r})', + f'some_other_arch (most recent profile was from ' + f'{profdata_info_2.date} at {profdata_info_2.location!r})' + ]), + tiny_render.line_break, + tiny_render.line_break, + 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' + 'Their status can be found at ', + tiny_render.Link( + href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + ), + '.', + ])) + + @unittest.mock.patch.object(subprocess, 'run') + def test_fetching_profdata_functions(self, subprocess_run_mock): + ls_return_value = unittest.mock.MagicMock() + ls_return_value.stdout = '\n'.join(( + ' 1234 2020-06-26T05:26:40Z gs://bar', + ' 44 2020-06-23T05:26:40Z gs://foo', + ' 1234 2020-06-25T05:26:40Z gs://zzz', + )) + subprocess_run_mock.return_value = ls_return_value + + most_recent = monitor_pgo_profiles.fetch_most_recent_profdata('arm') + self.assertEqual( + most_recent, + monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 6, 26, 5, 26, 40), + location='gs://bar', + )) + + +if __name__ == '__main__': + unittest.main() diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py new file mode 100755 index 00000000..3c0ad012 --- /dev/null +++ b/rust_tools/rust_uprev.py @@ -0,0 +1,673 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tool to automatically generate a new Rust uprev CL. + +This tool is intended to automatically generate a CL to uprev Rust to a +newer version in Chrome OS, including creating a new Rust version or +removing an old version. It's based on +src/third_party/chromiumos-overlay/dev-lang/rust/UPGRADE.md. When using +the tool, the progress can be saved to a JSON file, so the user can resume +the process after a failing step is fixed. Example usage to create a new +version: + +1. (inside chroot) $ ./rust_tools/rust_uprev.py + --state_file /tmp/state-file.json + create --rust_version 1.45.0 +2. Step "compile rust" failed due to the patches can't apply to new version +3. Manually fix the patches +4. Execute the command in step 1 again. +5. Iterate 1-4 for each failed step until the tool passes. + +Replace `create --rust_version 1.45.0` with `remove --rust_version 1.43.0` +if you want to remove all 1.43.0 related stuff in the same CL. Remember to +use a different state file if you choose to run different subcommands. + +If you want a hammer that can do everything for you, use the subcommand +`roll`. It can create a Rust uprev CL with `create` and `remove` and upload +the CL to chromium code review. + +See `--help` for all available options. +""" + +# pylint: disable=cros-logging-import + +import argparse +import pathlib +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple + +from llvm_tools import chroot, git +RUST_PATH = '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust' + + +def get_command_output(command: List[str], *args, **kwargs) -> str: + return subprocess.check_output( + command, encoding='utf-8', *args, **kwargs).strip() + + +class RustVersion(NamedTuple): + """NamedTuple represents a Rust version""" + major: int + minor: int + patch: int + + def __str__(self): + return f'{self.major}.{self.minor}.{self.patch}' + + @staticmethod + def parse_from_ebuild(ebuild_name: str) -> 'RustVersion': + input_re = re.compile(r'^rust-' + r'(?P<major>\d+)\.' + r'(?P<minor>\d+)\.' + r'(?P<patch>\d+)' + r'(:?-r\d+)?' + r'\.ebuild$') + m = input_re.match(ebuild_name) + assert m, f'failed to parse {ebuild_name!r}' + return RustVersion( + int(m.group('major')), int(m.group('minor')), int(m.group('patch'))) + + @staticmethod + def parse(x: str) -> 'RustVersion': + input_re = re.compile(r'^(?:rust-)?' + r'(?P<major>\d+)\.' + r'(?P<minor>\d+)\.' + r'(?P<patch>\d+)' + r'(?:.ebuild)?$') + m = input_re.match(x) + assert m, f'failed to parse {x!r}' + return RustVersion( + int(m.group('major')), int(m.group('minor')), int(m.group('patch'))) + + +def parse_commandline_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_file', + required=True, + help='A state file to hold previous completed steps. If the file ' + 'exists, it needs to be used together with --continue or --restart. ' + 'If not exist (do not use --continue in this case), we will create a ' + 'file for you.', + ) + parser.add_argument( + '--restart', + action='store_true', + help='Restart from the first step. Ignore the completed steps in ' + 'the state file', + ) + parser.add_argument( + '--continue', + dest='cont', + action='store_true', + help='Continue the steps from the state file', + ) + + create_parser_template = argparse.ArgumentParser(add_help=False) + create_parser_template.add_argument( + '--template', + type=RustVersion.parse, + default=None, + help='A template to use for creating a Rust uprev from, in the form ' + 'a.b.c The ebuild has to exist in the chroot. If not specified, the ' + 'tool will use the current Rust version in the chroot as template.', + ) + create_parser_template.add_argument( + '--skip_compile', + action='store_true', + help='Skip compiling rust to test the tool. Only for testing', + ) + + subparsers = parser.add_subparsers(dest='subparser_name') + subparser_names = [] + subparser_names.append('create') + create_parser = subparsers.add_parser( + 'create', + parents=[create_parser_template], + help='Create changes uprevs Rust to a new version', + ) + create_parser.add_argument( + '--rust_version', + type=RustVersion.parse, + required=True, + help='Rust version to uprev to, in the form a.b.c', + ) + + subparser_names.append('remove') + remove_parser = subparsers.add_parser( + 'remove', + help='Clean up old Rust version from chroot', + ) + remove_parser.add_argument( + '--rust_version', + type=RustVersion.parse, + default=None, + help='Rust version to remove, in the form a.b.c If not ' + 'specified, the tool will remove the oldest version in the chroot', + ) + + subparser_names.append('roll') + roll_parser = subparsers.add_parser( + 'roll', + parents=[create_parser_template], + help='A command can create and upload a Rust uprev CL, including ' + 'preparing the repo, creating new Rust uprev, deleting old uprev, ' + 'and upload a CL to crrev.', + ) + roll_parser.add_argument( + '--uprev', + type=RustVersion.parse, + required=True, + help='Rust version to uprev to, in the form a.b.c', + ) + roll_parser.add_argument( + '--remove', + type=RustVersion.parse, + default=None, + help='Rust version to remove, in the form a.b.c If not ' + 'specified, the tool will remove the oldest version in the chroot', + ) + roll_parser.add_argument( + '--skip_cross_compiler', + action='store_true', + help='Skip updating cross-compiler in the chroot', + ) + roll_parser.add_argument( + '--no_upload', + action='store_true', + help='If specified, the tool will not upload the CL for review', + ) + + args = parser.parse_args() + if args.subparser_name not in subparser_names: + parser.error('one of %s must be specified' % subparser_names) + + if args.cont and args.restart: + parser.error('Please select either --continue or --restart') + + if os.path.exists(args.state_file): + if not args.cont and not args.restart: + parser.error('State file exists, so you should either --continue ' + 'or --restart') + if args.cont and not os.path.exists(args.state_file): + parser.error('Indicate --continue but the state file does not exist') + + if args.restart and os.path.exists(args.state_file): + os.remove(args.state_file) + + return args + + +def parse_stage0_file(new_version: RustVersion) -> Tuple[str, str, str]: + # Find stage0 date, rustc and cargo + stage0_file = get_command_output([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{new_version}/src/stage0.txt' + ]) + regexp = re.compile(r'date:\s*(?P<date>\d+-\d+-\d+)\s+' + r'rustc:\s*(?P<rustc>\d+\.\d+\.\d+)\s+' + r'cargo:\s*(?P<cargo>\d+\.\d+\.\d+)') + m = regexp.search(stage0_file) + assert m, 'failed to parse stage0.txt file' + stage0_date, stage0_rustc, stage0_cargo = m.groups() + logging.info('Found stage0 file has date: %s, rustc: %s, cargo: %s', + stage0_date, stage0_rustc, stage0_cargo) + return stage0_date, stage0_rustc, stage0_cargo + + +def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] + ) -> Optional[Tuple[RustVersion, str]]: + if template is None: + ebuild_path = get_command_output(['equery', 'w', 'rust']) + ebuild_name = os.path.basename(ebuild_path) + template_version = RustVersion.parse_from_ebuild(ebuild_name) + else: + ebuild_path = find_ebuild_for_rust_version(template) + template_version = template + + if rust_version <= template_version: + logging.info( + 'Requested version %s is not newer than the template version %s.', + rust_version, template_version) + return None + + logging.info('Template Rust version is %s (ebuild: %r)', template_version, + ebuild_path) + return template_version, ebuild_path + + +def copy_patches(template_version: RustVersion, + new_version: RustVersion) -> None: + patch_path = os.path.join(RUST_PATH, 'files') + for f in os.listdir(patch_path): + if f'rust-{template_version}' not in f: + continue + logging.info('Rename patch %s to new version', f) + new_name = f.replace(str(template_version), str(new_version)) + shutil.copyfile( + os.path.join(patch_path, f), + os.path.join(patch_path, new_name), + ) + + subprocess.check_call(['git', 'add', f'files/rust-{new_version}-*.patch'], + cwd=RUST_PATH) + + +def create_ebuild(template_ebuild: str, new_version: RustVersion) -> str: + shutil.copyfile(template_ebuild, + os.path.join(RUST_PATH, f'rust-{new_version}.ebuild')) + subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], + cwd=RUST_PATH) + return os.path.join(RUST_PATH, f'rust-{new_version}.ebuild') + + +def update_ebuild(ebuild_file: str, stage0_info: Tuple[str, str, str]) -> None: + stage0_date, stage0_rustc, stage0_cargo = stage0_info + with open(ebuild_file, encoding='utf-8') as f: + contents = f.read() + # Update STAGE0_DATE in the ebuild + stage0_date_re = re.compile(r'STAGE0_DATE="(\d+-\d+-\d+)"') + if not stage0_date_re.search(contents): + raise RuntimeError('STAGE0_DATE not found in rust ebuild') + new_contents = stage0_date_re.sub(f'STAGE0_DATE="{stage0_date}"', contents) + + # Update STAGE0_VERSION in the ebuild + stage0_rustc_re = re.compile(r'STAGE0_VERSION="[^"]*"') + if not stage0_rustc_re.search(new_contents): + raise RuntimeError('STAGE0_VERSION not found in rust ebuild') + new_contents = stage0_rustc_re.sub(f'STAGE0_VERSION="{stage0_rustc}"', + new_contents) + + # Update STAGE0_VERSION_CARGO in the ebuild + stage0_cargo_re = re.compile(r'STAGE0_VERSION_CARGO="[^"]*"') + if not stage0_cargo_re.search(new_contents): + raise RuntimeError('STAGE0_VERSION_CARGO not found in rust ebuild') + new_contents = stage0_cargo_re.sub(f'STAGE0_VERSION_CARGO="{stage0_cargo}"', + new_contents) + with open(ebuild_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + logging.info( + 'Rust ebuild file has STAGE0_DATE, STAGE0_VERSION, STAGE0_VERSION_CARGO ' + 'updated to %s, %s, %s respectively', stage0_date, stage0_rustc, + stage0_cargo) + + +def flip_mirror_in_ebuild(ebuild_file: str, add: bool) -> None: + restrict_re = re.compile( + r'(?P<before>RESTRICT=")(?P<values>"[^"]*"|.*)(?P<after>")') + with open(ebuild_file, encoding='utf-8') as f: + contents = f.read() + m = restrict_re.search(contents) + assert m, 'failed to find RESTRICT variable in Rust ebuild' + values = m.group('values') + if add: + if 'mirror' in values: + return + values += ' mirror' + else: + if 'mirror' not in values: + return + values = values.replace(' mirror', '') + new_contents = restrict_re.sub(r'\g<before>%s\g<after>' % values, contents) + with open(ebuild_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + + +def rust_ebuild_actions(actions: List[str], sudo: bool = False) -> None: + ebuild_path_inchroot = get_command_output(['equery', 'w', 'rust']) + cmd = ['ebuild', ebuild_path_inchroot] + actions + if sudo: + cmd = ['sudo'] + cmd + subprocess.check_call(cmd) + + +def update_manifest(ebuild_file: str) -> None: + logging.info('Added "mirror" to RESTRICT to Rust ebuild') + flip_mirror_in_ebuild(ebuild_file, add=True) + rust_ebuild_actions(['manifest']) + logging.info('Removed "mirror" to RESTRICT from Rust ebuild') + flip_mirror_in_ebuild(ebuild_file, add=False) + + +def update_rust_packages(rust_version: RustVersion, add: bool) -> None: + package_file = os.path.join( + RUST_PATH, '../../profiles/targets/chromeos/package.provided') + with open(package_file, encoding='utf-8') as f: + contents = f.read() + if add: + rust_packages_re = re.compile(r'dev-lang/rust-(\d+\.\d+\.\d+)') + rust_packages = rust_packages_re.findall(contents) + # Assume all the rust packages are in alphabetical order, so insert the new + # version to the place after the last rust_packages + new_str = f'dev-lang/rust-{rust_version}' + new_contents = contents.replace(rust_packages[-1], + f'{rust_packages[-1]}\n{new_str}') + logging.info('%s has been inserted into package.provided', new_str) + else: + old_str = f'dev-lang/rust-{rust_version}\n' + assert old_str in contents, f'{old_str!r} not found in package.provided' + new_contents = contents.replace(old_str, '') + logging.info('%s has been removed from package.provided', old_str) + + with open(package_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + + +def update_virtual_rust(template_version: RustVersion, + new_version: RustVersion) -> None: + virtual_rust_dir = os.path.join(RUST_PATH, '../../virtual/rust') + assert os.path.exists(virtual_rust_dir) + shutil.copyfile( + os.path.join(virtual_rust_dir, f'rust-{template_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{new_version}.ebuild')) + subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], + cwd=virtual_rust_dir) + + +def upload_single_tarball(rust_url: str, tarfile_name: str, + tempdir: str) -> None: + rust_src = f'{rust_url}/{tarfile_name}' + gsutil_location = f'gs://chromeos-localmirror/distfiles/{tarfile_name}' + + missing_file = subprocess.call( + ['gsutil', 'ls', gsutil_location], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not missing_file: + logging.info('Rust artifact at %s already exists; skipping download', + gsutil_location) + return + + logging.info('Downloading Rust artifact from %s', rust_src) + + # Download Rust's source + rust_file = os.path.join(tempdir, tarfile_name) + subprocess.check_call(['curl', '-f', '-o', rust_file, rust_src]) + + # Verify the signature of the source + sig_file = os.path.join(tempdir, 'rustc_sig.asc') + subprocess.check_call(['curl', '-f', '-o', sig_file, f'{rust_src}.asc']) + try: + subprocess.check_output(['gpg', '--verify', sig_file, rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if "gpg: Can't check signature" not in e.output: + raise RuntimeError(f'Failed to execute `gpg --verify`, {e.output}') + + # If it fails to verify the signature, try import rustc key, and retry. + keys = get_command_output( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + subprocess.run(['gpg', '--import'], + input=keys, + encoding='utf-8', + check=True) + subprocess.check_call(['gpg', '--verify', sig_file, rust_file]) + + # Since we are using `-n` to skip an item if it already exists, there's no + # need to check if the file exists on GS bucket or not. + subprocess.check_call( + ['gsutil', 'cp', '-n', '-a', 'public-read', rust_file, gsutil_location]) + + +def upload_to_localmirror(tempdir: str, rust_version: RustVersion, + stage0_info: Tuple[str, str, str]) -> None: + stage0_date, stage0_rustc, stage0_cargo = stage0_info + rust_url = 'https://static.rust-lang.org/dist' + # Upload rustc source + upload_single_tarball( + rust_url, + f'rustc-{rust_version}-src.tar.gz', + tempdir, + ) + # Upload stage0 toolchain + upload_single_tarball( + f'{rust_url}/{stage0_date}', + f'rust-std-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + # Upload stage0 source + upload_single_tarball( + rust_url, + f'rustc-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + # Upload stage0 cargo + upload_single_tarball( + rust_url, + f'cargo-{stage0_cargo}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + + +def perform_step(state_file: pathlib.Path, + tmp_state_file: pathlib.Path, + completed_steps: Dict[str, Any], + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None) -> T: + if step_name in completed_steps: + logging.info('Skipping previously completed step %s', step_name) + if result_from_json: + return result_from_json(completed_steps[step_name]) + return completed_steps[step_name] + + logging.info('Running step %s', step_name) + val = step_fn() + logging.info('Step %s complete', step_name) + if result_to_json: + completed_steps[step_name] = result_to_json(val) + else: + completed_steps[step_name] = val + + with tmp_state_file.open('w', encoding='utf-8') as f: + json.dump(completed_steps, f, indent=4) + tmp_state_file.rename(state_file) + return val + + +def prepare_uprev_from_json(obj: Any) -> Optional[Tuple[RustVersion, str]]: + if not obj: + return None + version, ebuild_path = obj + return RustVersion(*version), ebuild_path + + +def create_rust_uprev(rust_version: RustVersion, + maybe_template_version: Optional[RustVersion], + skip_compile: bool, run_step: Callable[[], T]) -> None: + stage0_info = run_step( + 'parse stage0 file', lambda: parse_stage0_file(rust_version)) + template_version, template_ebuild = run_step( + 'prepare uprev', + lambda: prepare_uprev(rust_version, maybe_template_version), + result_from_json=prepare_uprev_from_json, + ) + if template_ebuild is None: + return + + run_step('copy patches', lambda: copy_patches(template_version, rust_version)) + ebuild_file = run_step( + 'create ebuild', lambda: create_ebuild(template_ebuild, rust_version)) + run_step('update ebuild', lambda: update_ebuild(ebuild_file, stage0_info)) + with tempfile.TemporaryDirectory(dir='/tmp') as tempdir: + run_step('upload_to_localmirror', lambda: upload_to_localmirror( + tempdir, rust_version, stage0_info)) + run_step('update manifest to add new version', lambda: update_manifest( + ebuild_file)) + if not skip_compile: + run_step('emerge rust', lambda: subprocess.check_call( + ['sudo', 'emerge', 'dev-lang/rust'])) + run_step('insert version into rust packages', lambda: update_rust_packages( + rust_version, add=True)) + run_step('upgrade virtual/rust', lambda: update_virtual_rust( + template_version, rust_version)) + + +def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]: + return [(RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) + for x in os.listdir(RUST_PATH) + if x.endswith('.ebuild')] + + +def find_oldest_rust_version_in_chroot() -> Tuple[RustVersion, str]: + rust_versions = find_rust_versions_in_chroot() + if len(rust_versions) <= 1: + raise RuntimeError('Expect to find more than one Rust versions') + return min(rust_versions) + + +def find_ebuild_for_rust_version(version: RustVersion) -> str: + rust_ebuilds = [ + ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version + ] + if not rust_ebuilds: + raise ValueError(f'No Rust ebuilds found matching {version}') + if len(rust_ebuilds) > 1: + raise ValueError(f'Multiple Rust ebuilds found matching {version}: ' + f'{rust_ebuilds}') + return rust_ebuilds[0] + + +def remove_files(filename: str, path: str) -> None: + subprocess.check_call(['git', 'rm', filename], cwd=path) + + +def remove_rust_uprev(rust_version: Optional[RustVersion], + run_step: Callable[[], T]) -> None: + + def find_desired_rust_version(): + if rust_version: + return rust_version, find_ebuild_for_rust_version(rust_version) + return find_oldest_rust_version_in_chroot() + + delete_version, delete_ebuild = run_step( + 'find rust version to delete', + find_desired_rust_version, + result_from_json=prepare_uprev_from_json, + ) + run_step( + 'remove patches', lambda: remove_files( + f'files/rust-{delete_version}-*.patch', RUST_PATH)) + run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH)) + ebuild_file = get_command_output(['equery', 'w', 'rust']) + run_step('update manifest to delete old version', lambda: update_manifest( + ebuild_file)) + run_step('remove version from rust packages', lambda: update_rust_packages( + delete_version, add=False)) + run_step( + 'remove virtual/rust', lambda: remove_files( + f'rust-{delete_version}.ebuild', + os.path.join(RUST_PATH, '../../virtual/rust'))) + + +def create_new_repo(rust_version: RustVersion) -> None: + output = get_command_output(['git', 'status', '--porcelain'], cwd=RUST_PATH) + if output: + raise RuntimeError( + f'{RUST_PATH} has uncommitted changes, please either discard them ' + 'or commit them.') + git.CreateBranch(RUST_PATH, f'rust-to-{rust_version}') + + +def build_cross_compiler() -> None: + # Get target triples in ebuild + rust_ebuild = get_command_output(['equery', 'w', 'rust']) + with open(rust_ebuild, encoding='utf-8') as f: + contents = f.read() + + target_triples_re = re.compile(r'RUSTC_TARGET_TRIPLES=\(([^)]+)\)') + m = target_triples_re.search(contents) + assert m, 'RUST_TARGET_TRIPLES not found in rust ebuild' + target_triples = m.group(1).strip().split('\n') + + compiler_targets_to_install = [ + target.strip() for target in target_triples if 'cros-' in target + ] + for target in target_triples: + if 'cros-' not in target: + continue + target = target.strip() + + # We also always need arm-none-eabi, though it's not mentioned in + # RUSTC_TARGET_TRIPLES. + compiler_targets_to_install.append('arm-none-eabi') + + logging.info('Emerging cross compilers %s', compiler_targets_to_install) + subprocess.check_call( + ['sudo', 'emerge', '-j', '-G'] + + [f'cross-{target}/gcc' for target in compiler_targets_to_install]) + + +def create_new_commit(rust_version: RustVersion) -> None: + subprocess.check_call(['git', 'add', '-A'], cwd=RUST_PATH) + messages = [ + f'[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}', + '', + 'This CL is created by rust_uprev tool automatically.' + '', + 'BUG=None', + 'TEST=Use CQ to test the new Rust version', + ] + git.UploadChanges(RUST_PATH, f'rust-to-{rust_version}', messages) + + +def main() -> None: + if not chroot.InChroot(): + raise RuntimeError('This script must be executed inside chroot') + + logging.basicConfig(level=logging.INFO) + + args = parse_commandline_args() + + state_file = pathlib.Path(args.state_file) + tmp_state_file = state_file.with_suffix('.tmp') + + try: + with state_file.open(encoding='utf-8') as f: + completed_steps = json.load(f) + except FileNotFoundError: + completed_steps = {} + + def run_step( + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, + ) -> T: + return perform_step(state_file, tmp_state_file, completed_steps, step_name, + step_fn, result_from_json, result_to_json) + + if args.subparser_name == 'create': + create_rust_uprev(args.rust_version, args.template, args.skip_compile, + run_step) + elif args.subparser_name == 'remove': + remove_rust_uprev(args.rust_version, run_step) + else: + # If you have added more subparser_name, please also add the handlers above + assert args.subparser_name == 'roll' + run_step('create new repo', lambda: create_new_repo(args.uprev)) + if not args.skip_cross_compiler: + run_step('build cross compiler', build_cross_compiler) + create_rust_uprev(args.uprev, args.template, args.skip_compile, run_step) + remove_rust_uprev(args.remove, run_step) + if not args.no_upload: + run_step('create rust uprev CL', lambda: create_new_commit(args.uprev)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py new file mode 100755 index 00000000..fc506004 --- /dev/null +++ b/rust_tools/rust_uprev_test.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for rust_uprev.py""" + +# pylint: disable=cros-logging-import +import os +import shutil +import subprocess +import unittest +from unittest import mock + +from llvm_tools import git + +import rust_uprev + + +class RustVersionTest(unittest.TestCase): + """Tests for RustVersion class""" + + def test_str(self): + obj = rust_uprev.RustVersion(major=1, minor=2, patch=3) + self.assertEqual(str(obj), '1.2.3') + + def test_parse_version_only(self): + expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) + actual = rust_uprev.RustVersion.parse('1.2.3') + self.assertEqual(expected, actual) + + def test_parse_ebuild_name(self): + expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) + actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3.ebuild') + self.assertEqual(expected, actual) + + actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3-r1.ebuild') + self.assertEqual(expected, actual) + + def test_parse_fail(self): + with self.assertRaises(AssertionError) as context: + rust_uprev.RustVersion.parse('invalid-rust-1.2.3') + self.assertEqual("failed to parse 'invalid-rust-1.2.3'", + str(context.exception)) + + +class PrepareUprevTest(unittest.TestCase): + """Tests for prepare_uprev step in rust_uprev""" + + def setUp(self): + self.version_old = rust_uprev.RustVersion(1, 2, 3) + self.version_new = rust_uprev.RustVersion(1, 3, 5) + + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_success_with_template(self, mock_command, mock_find_ebuild): + expected = (self.version_old, '/path/to/ebuild') + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=self.version_old) + self.assertEqual(expected, actual) + mock_command.assert_not_called() + + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_return_none_with_template_larger_than_input(self, mock_command, + _mock_find_ebuild): + ret = rust_uprev.prepare_uprev( + rust_version=self.version_old, template=self.version_new) + self.assertIsNone(ret) + mock_command.assert_not_called() + + @mock.patch.object(os.path, 'exists') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_success_without_template(self, mock_command, mock_exists): + rust_ebuild_path = f'/path/to/rust/rust-{self.version_old}-r3.ebuild' + mock_command.return_value = rust_ebuild_path + expected = (self.version_old, rust_ebuild_path) + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=None) + self.assertEqual(expected, actual) + mock_command.assert_called_once_with(['equery', 'w', 'rust']) + mock_exists.assert_not_called() + + @mock.patch.object(os.path, 'exists') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_return_none_with_ebuild_larger_than_input(self, mock_command, + mock_exists): + mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild' + ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=None) + self.assertIsNone(ret) + mock_exists.assert_not_called() + + def test_prepare_uprev_from_json(self): + ebuild_path = '/path/to/the/ebuild' + json_result = (list(self.version_new), ebuild_path) + expected = (self.version_new, ebuild_path) + actual = rust_uprev.prepare_uprev_from_json(json_result) + self.assertEqual(expected, actual) + + +class UpdateEbuildTest(unittest.TestCase): + """Tests for update_ebuild step in rust_uprev""" + ebuild_file_before = """ + STAGE0_DATE="2019-01-01" + STAGE0_VERSION="any.random.(number)" + STAGE0_VERSION_CARGO="0.0.0" + """ + ebuild_file_after = """ + STAGE0_DATE="2020-01-01" + STAGE0_VERSION="1.1.1" + STAGE0_VERSION_CARGO="0.1.0" + """ + + def test_success(self): + mock_open = mock.mock_open(read_data=self.ebuild_file_before) + ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' + with mock.patch('builtins.open', mock_open): + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) + mock_open.return_value.__enter__().write.assert_called_once_with( + self.ebuild_file_after) + + def test_fail_when_ebuild_misses_a_variable(self): + ebuild_file = 'STAGE0_DATE="2019-01-01"' + mock_open = mock.mock_open(read_data=ebuild_file) + ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' + with mock.patch('builtins.open', mock_open): + with self.assertRaises(RuntimeError) as context: + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) + self.assertEqual('STAGE0_VERSION not found in rust ebuild', + str(context.exception)) + + +class UpdateManifestTest(unittest.TestCase): + """Tests for update_manifest step in rust_uprev""" + + # pylint: disable=protected-access + def _run_test_flip_mirror(self, before, after, add, expect_write): + mock_open = mock.mock_open(read_data=f'RESTRICT="{before}"') + with mock.patch('builtins.open', mock_open): + rust_uprev.flip_mirror_in_ebuild('', add=add) + if expect_write: + mock_open.return_value.__enter__().write.assert_called_once_with( + f'RESTRICT="{after}"') + + def test_add_mirror_in_ebuild(self): + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2 mirror', + add=True, + expect_write=True) + + def test_remove_mirror_in_ebuild(self): + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2', + add=False, + expect_write=True) + + def test_add_mirror_when_exists(self): + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2 mirror', + add=True, + expect_write=False) + + def test_remove_mirror_when_not_exists(self): + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2', + add=False, + expect_write=False) + + @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild') + @mock.patch.object(rust_uprev, 'rust_ebuild_actions') + def test_update_manifest(self, mock_run, mock_flip): + ebuild_file = '/path/to/rust/rust-1.1.1.ebuild' + rust_uprev.update_manifest(ebuild_file) + mock_run.assert_called_once_with(['manifest']) + mock_flip.assert_has_calls( + [mock.call(ebuild_file, add=True), + mock.call(ebuild_file, add=False)]) + + +class UpdateRustPackagesTests(unittest.TestCase): + """Tests for update_rust_packages step.""" + + def setUp(self): + self.old_version = rust_uprev.RustVersion(1, 1, 0) + self.current_version = rust_uprev.RustVersion(1, 2, 3) + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, + 'rust-{self.new_version}.ebuild') + + def test_add_new_rust_packages(self): + package_before = (f'dev-lang/rust-{self.old_version}\n' + f'dev-lang/rust-{self.current_version}') + package_after = (f'dev-lang/rust-{self.old_version}\n' + f'dev-lang/rust-{self.current_version}\n' + f'dev-lang/rust-{self.new_version}') + mock_open = mock.mock_open(read_data=package_before) + with mock.patch('builtins.open', mock_open): + rust_uprev.update_rust_packages(self.new_version, add=True) + mock_open.return_value.__enter__().write.assert_called_once_with( + package_after) + + def test_remove_old_rust_packages(self): + package_before = (f'dev-lang/rust-{self.old_version}\n' + f'dev-lang/rust-{self.current_version}\n' + f'dev-lang/rust-{self.new_version}') + package_after = (f'dev-lang/rust-{self.current_version}\n' + f'dev-lang/rust-{self.new_version}') + mock_open = mock.mock_open(read_data=package_before) + with mock.patch('builtins.open', mock_open): + rust_uprev.update_rust_packages(self.old_version, add=False) + mock_open.return_value.__enter__().write.assert_called_once_with( + package_after) + + +class UploadToLocalmirrorTests(unittest.TestCase): + """Tests for upload_to_localmirror""" + + def setUp(self): + self.tempdir = '/tmp/any/dir' + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.rust_url = 'https://static.rust-lang.org/dist' + self.tarfile_name = f'rustc-{self.new_version}-src.tar.gz' + self.rust_src = f'https://static.rust-lang.org/dist/{self.tarfile_name}' + self.gsurl = f'gs://chromeos-localmirror/distfiles/{self.tarfile_name}' + self.rust_file = os.path.join(self.tempdir, self.tarfile_name) + self.sig_file = os.path.join(self.tempdir, 'rustc_sig.asc') + + @mock.patch.object(subprocess, 'call', return_value=1) + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + def test_pass_without_retry(self, mock_run, mock_output, mock_call, + mock_raw_call): + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_output.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + mock_run.assert_not_called() + + @mock.patch.object(subprocess, 'call') + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call, + mock_raw_call): + mock_check.side_effect = subprocess.CalledProcessError( + returncode=2, cmd=None, output="gpg: Can't check signature") + mock_output.return_value = 'some_gpg_keys' + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_check.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_output.assert_called_once_with( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + mock_run.assert_called_once_with(['gpg', '--import'], + input='some_gpg_keys', + encoding='utf-8', + check=True) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call(['gpg', '--verify', self.sig_file, self.rust_file]), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + + @mock.patch.object(rust_uprev, 'upload_single_tarball') + def test_upload_to_mirror(self, mock_upload): + stage0_info = '2020-01-01', '1.1.1', '0.1.0' + rust_uprev.upload_to_localmirror(self.tempdir, self.new_version, + stage0_info) + mock_upload.assert_has_calls([ + mock.call(self.rust_url, f'rustc-{self.new_version}-src.tar.gz', + self.tempdir), + mock.call(f'{self.rust_url}/{stage0_info[0]}', + f'rust-std-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'rustc-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'cargo-{stage0_info[2]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + ]) + + +class RustUprevOtherStagesTests(unittest.TestCase): + """Tests for other steps in rust_uprev""" + + def setUp(self): + self.old_version = rust_uprev.RustVersion(1, 1, 0) + self.current_version = rust_uprev.RustVersion(1, 2, 3) + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, + 'rust-{self.new_version}.ebuild') + + @mock.patch.object(rust_uprev, 'get_command_output') + def test_parse_stage0_file(self, mock_get): + stage0_file = """ + unrelated stuff before + date: 2020-01-01 + rustc: 1.1.1 + cargo: 0.1.0 + unrelated stuff after + """ + mock_get.return_value = stage0_file + expected = '2020-01-01', '1.1.1', '0.1.0' + rust_version = rust_uprev.RustVersion(1, 2, 3) + actual = rust_uprev.parse_stage0_file(rust_version) + self.assertEqual(expected, actual) + mock_get.assert_called_once_with([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{rust_version}/src/stage0.txt' + ]) + + @mock.patch.object(shutil, 'copyfile') + @mock.patch.object(os, 'listdir') + @mock.patch.object(subprocess, 'check_call') + def test_copy_patches(self, mock_call, mock_ls, mock_copy): + mock_ls.return_value = [ + f'rust-{self.old_version}-patch-1.patch', + f'rust-{self.old_version}-patch-2-old.patch', + f'rust-{self.current_version}-patch-1.patch', + f'rust-{self.current_version}-patch-2-new.patch' + ] + rust_uprev.copy_patches(self.current_version, self.new_version) + mock_copy.assert_has_calls([ + mock.call( + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.current_version}-patch-1.patch'), + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.new_version}-patch-1.patch'), + ), + mock.call( + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.current_version}-patch-2-new.patch'), + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.new_version}-patch-2-new.patch')) + ]) + mock_call.assert_called_once_with( + ['git', 'add', f'files/rust-{self.new_version}-*.patch'], + cwd=rust_uprev.RUST_PATH) + + @mock.patch.object(shutil, 'copyfile') + @mock.patch.object(subprocess, 'check_call') + def test_create_ebuild(self, mock_call, mock_copy): + template_ebuild = f'/path/to/rust-{self.current_version}-r2.ebuild' + rust_uprev.create_ebuild(template_ebuild, self.new_version) + mock_copy.assert_called_once_with( + template_ebuild, + os.path.join(rust_uprev.RUST_PATH, f'rust-{self.new_version}.ebuild')) + mock_call.assert_called_once_with( + ['git', 'add', f'rust-{self.new_version}.ebuild'], + cwd=rust_uprev.RUST_PATH) + + @mock.patch.object(os.path, 'exists', return_value=True) + @mock.patch.object(shutil, 'copyfile') + @mock.patch.object(subprocess, 'check_call') + def test_update_virtual_rust(self, mock_call, mock_copy, mock_exists): + virtual_rust_dir = os.path.join(rust_uprev.RUST_PATH, '../../virtual/rust') + rust_uprev.update_virtual_rust(self.current_version, self.new_version) + mock_call.assert_called_once_with( + ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=virtual_rust_dir) + mock_copy.assert_called_once_with( + os.path.join(virtual_rust_dir, f'rust-{self.current_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{self.new_version}.ebuild')) + mock_exists.assert_called_once_with(virtual_rust_dir) + + @mock.patch.object(os, 'listdir') + def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): + oldest_version_name = f'rust-{self.old_version}.ebuild' + mock_ls.return_value = [ + oldest_version_name, f'rust-{self.current_version}.ebuild', + f'rust-{self.new_version}.ebuild' + ] + actual = rust_uprev.find_oldest_rust_version_in_chroot() + expected = (self.old_version, + os.path.join(rust_uprev.RUST_PATH, oldest_version_name)) + self.assertEqual(expected, actual) + + @mock.patch.object(os, 'listdir') + def test_find_oldest_rust_version_in_chroot_fail_with_only_one_ebuild( + self, mock_ls): + mock_ls.return_value = [f'rust-{self.new_version}.ebuild'] + with self.assertRaises(RuntimeError) as context: + rust_uprev.find_oldest_rust_version_in_chroot() + self.assertEqual('Expect to find more than one Rust versions', + str(context.exception)) + + @mock.patch.object(rust_uprev, 'get_command_output') + @mock.patch.object(git, 'CreateBranch') + def test_create_new_repo(self, mock_branch, mock_output): + mock_output.return_value = '' + rust_uprev.create_new_repo(self.new_version) + mock_branch.assert_called_once_with(rust_uprev.RUST_PATH, + f'rust-to-{self.new_version}') + + @mock.patch.object(rust_uprev, 'get_command_output') + @mock.patch.object(subprocess, 'check_call') + def test_build_cross_compiler(self, mock_call, mock_output): + mock_output.return_value = f'rust-{self.new_version}.ebuild' + cros_targets = [ + 'x86_64-cros-linux-gnu', + 'armv7a-cros-linux-gnueabihf', + 'aarch64-cros-linux-gnu', + ] + all_triples = ['x86_64-pc-linux-gnu'] + cros_targets + rust_ebuild = 'RUSTC_TARGET_TRIPLES=(' + '\n\t'.join(all_triples) + ')' + mock_open = mock.mock_open(read_data=rust_ebuild) + with mock.patch('builtins.open', mock_open): + rust_uprev.build_cross_compiler() + + mock_call.assert_called_once_with( + ['sudo', 'emerge', '-j', '-G'] + + [f'cross-{x}/gcc' for x in cros_targets + ['arm-none-eabi']]) + + +if __name__ == '__main__': + unittest.main() diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py new file mode 100755 index 00000000..b9ad7b82 --- /dev/null +++ b/rust_tools/rust_watch.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Checks for various upstream events with the Rust toolchain. + +Sends an email if something interesting (probably) happened. +""" + +# pylint: disable=cros-logging-import + +import argparse +import itertools +import json +import logging +import pathlib +import re +import shutil +import subprocess +import sys +import time +from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple + +from cros_utils import email_sender +from cros_utils import tiny_render + + +def gentoo_sha_to_link(sha: str) -> str: + """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" + return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' + + +def send_email(subject: str, body: List[tiny_render.Piece]) -> None: + """Sends an email with the given title and body to... whoever cares.""" + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier='rust-watch', + well_known_recipients=['cros-team'], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) + + +class RustReleaseVersion(NamedTuple): + """Represents a version of Rust's stable compiler.""" + major: int + minor: int + patch: int + + @staticmethod + def from_string(version_string: str) -> 'RustReleaseVersion': + m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) + if not m: + raise ValueError(f"{version_string!r} isn't a valid version string") + return RustReleaseVersion(*[int(x) for x in m.groups()]) + + def __str__(self) -> str: + return f'{self.major}.{self.minor}.{self.patch}' + + def to_json(self) -> str: + return str(self) + + @staticmethod + def from_json(s: str) -> 'RustReleaseVersion': + return RustReleaseVersion.from_string(s) + + +class State(NamedTuple): + """State that we keep around from run to run.""" + # The last Rust release tag that we've seen. + last_seen_release: RustReleaseVersion + + # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen + # that updates it. + last_gentoo_sha: str + + def to_json(self) -> Dict[str, Any]: + return { + 'last_seen_release': self.last_seen_release.to_json(), + 'last_gentoo_sha': self.last_gentoo_sha, + } + + @staticmethod + def from_json(s: Dict[str, Any]) -> 'State': + return State( + last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), + last_gentoo_sha=s['last_gentoo_sha'], + ) + + +def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: + """Parses `git ls-remote --tags` output into Rust stable release versions.""" + refs_tags = 'refs/tags/' + for line in lines: + _sha, tag = line.split(None, 1) + tag = tag.strip() + # Each tag has an associated 'refs/tags/name^{}', which is the actual + # object that the tag points to. That's irrelevant to us. + if tag.endswith('^{}'): + continue + + if not tag.startswith(refs_tags): + continue + + short_tag = tag[len(refs_tags):] + # There are a few old versioning schemes. Ignore them. + if short_tag.startswith('0.') or short_tag.startswith('release-'): + continue + yield RustReleaseVersion.from_string(short_tag) + + +def fetch_most_recent_release() -> RustReleaseVersion: + """Fetches the most recent stable `rustc` version.""" + result = subprocess.run( + ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + tag_lines = result.stdout.strip().splitlines() + return max(parse_release_tags(tag_lines)) + + +class GitCommit(NamedTuple): + """Represents a single git commit.""" + sha: str + subject: str + + +def update_git_repo(git_dir: pathlib.Path) -> None: + """Updates the repo at `git_dir`, retrying a few times on failure.""" + for i in itertools.count(start=1): + result = subprocess.run( + ['git', 'fetch', 'origin'], + check=False, + cwd=str(git_dir), + stdin=None, + ) + + if not result.returncode: + break + + if i == 5: + # 5 attempts is too many. Something else may be wrong. + result.check_returncode() + + sleep_time = 60 * i + logging.error("Failed updating gentoo's repo; will try again in %ds...", + sleep_time) + time.sleep(sleep_time) + + +def get_new_gentoo_commits(git_dir: pathlib.Path, + most_recent_sha: str) -> List[GitCommit]: + """Gets commits to dev-lang/rust since `most_recent_sha`. + + Older commits come earlier in the returned list. + """ + commits = subprocess.run( + [ + 'git', + 'log', + '--format=%H %s', + f'{most_recent_sha}..origin/master', + '--', + 'dev-lang/rust', + ], + capture_output=True, + check=False, + cwd=str(git_dir), + encoding='utf-8', + ) + + if commits.returncode: + logging.error('Error getting new gentoo commits; stderr:\n%s', + commits.stderr) + commits.check_returncode() + + results = [] + for line in commits.stdout.strip().splitlines(): + sha, subject = line.strip().split(None, 1) + results.append(GitCommit(sha=sha, subject=subject)) + + # `git log` outputs things in newest -> oldest order. + results.reverse() + return results + + +def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: + """Sets up a gentoo git repo at the given directory. Returns HEAD.""" + subprocess.run( + [ + 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', + str(git_dir) + ], + stdin=None, + check=True, + ) + + head_rev = subprocess.run( + ['git', 'rev-parse', 'HEAD'], + cwd=str(git_dir), + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + return head_rev.stdout.strip() + + +def read_state(state_file: pathlib.Path) -> State: + """Reads state from the given file.""" + with state_file.open(encoding='utf-8') as f: + return State.from_json(json.load(f)) + + +def atomically_write_state(state_file: pathlib.Path, state: State) -> None: + """Writes state to the given file.""" + temp_file = pathlib.Path(str(state_file) + '.new') + with temp_file.open('w', encoding='utf-8') as f: + json.dump(state.to_json(), f) + temp_file.rename(state_file) + + +def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, + new_gentoo_commits: List[GitCommit] + ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: + """Creates an email given our new state, if doing so is appropriate.""" + subject_pieces = [] + body_pieces = [] + + if newest_release > old_state.last_seen_release: + subject_pieces.append('new rustc release detected') + body_pieces.append(f'Rustc tag for v{newest_release} was found.') + + if new_gentoo_commits: + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] + + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) + + body_pieces.append(tiny_render.UnorderedList(commit_lines)) + + if not subject_pieces: + return None + + subject = '[rust-watch] ' + '; '.join(subject_pieces) + return subject, body_pieces + + +def main(argv: List[str]) -> None: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_dir', required=True, help='Directory to store state in.') + parser.add_argument( + '--skip_email', action='store_true', help="Don't send an email.") + parser.add_argument( + '--skip_state_update', + action='store_true', + help="Don't update the state file. Doesn't apply to initial setup.") + opts = parser.parse_args(argv) + + state_dir = pathlib.Path(opts.state_dir) + state_file = state_dir / 'state.json' + gentoo_subdir = state_dir / 'upstream-gentoo' + if not state_file.exists(): + logging.info("state_dir isn't fully set up; doing that now.") + + # Could be in a partially set-up state. + if state_dir.exists(): + logging.info('incomplete state_dir detected; removing.') + shutil.rmtree(str(state_dir)) + + state_dir.mkdir(parents=True) + most_recent_release = fetch_most_recent_release() + most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=most_recent_gentoo_commit, + ), + ) + # Running through this _should_ be a nop, but do it anyway. Should make any + # bugs more obvious on the first run of the script. + + prior_state = read_state(state_file) + logging.info('Last state was %r', prior_state) + + most_recent_release = fetch_most_recent_release() + logging.info('Most recent Rust release is %s', most_recent_release) + + logging.info('Fetching new commits from Gentoo') + update_git_repo(gentoo_subdir) + new_commits = get_new_gentoo_commits(gentoo_subdir, + prior_state.last_gentoo_sha) + logging.info('New commits: %r', new_commits) + + maybe_email = maybe_compose_email(prior_state, most_recent_release, + new_commits) + + if maybe_email is None: + logging.info('No updates to send') + else: + title, body = maybe_email + if opts.skip_email: + logging.info('Skipping sending email with title %r and contents\n%s', + title, tiny_render.render_html_pieces(body)) + else: + logging.info('Sending email') + send_email(title, body) + + if opts.skip_state_update: + logging.info('Skipping state update, as requested') + return + + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=newest_sha, + ), + ) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py new file mode 100755 index 00000000..97d111fc --- /dev/null +++ b/rust_tools/rust_watch_test.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for rust_watch.py.""" + +# pylint: disable=cros-logging-import + +import logging +import pathlib +import subprocess +import time +import unittest +import unittest.mock + +import rust_watch +from cros_utils import tiny_render + + +class Test(unittest.TestCase): + """Tests.""" + + def _silence_logs(self): + """Silences all log output until the end of the current test.""" + + def should_log(_record): + return 0 + + logger = logging.root + logger.addFilter(should_log) + self.addCleanup(logger.removeFilter, should_log) + + def test_release_version_parsing(self): + self.assertEqual( + rust_watch.RustReleaseVersion.from_string('1.2.3'), + rust_watch.RustReleaseVersion(1, 2, 3), + ) + + def test_release_version_json_round_trips(self): + ver = rust_watch.RustReleaseVersion(1, 2, 3) + self.assertEqual( + rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver) + + def test_state_json_round_trips(self): + state = rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3), + last_gentoo_sha='abc123', + ) + + self.assertEqual(rust_watch.State.from_json(state.to_json()), state) + + @unittest.mock.patch.object(subprocess, 'run') + @unittest.mock.patch.object(time, 'sleep') + def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock): + self._silence_logs() + + oh_no_error = ValueError('oh no') + + def check_returncode(): + raise oh_no_error + + run_call_count = 0 + + def run_sideeffect(*_args, **_kwargs): + nonlocal run_call_count + run_call_count += 1 + result = unittest.mock.Mock() + result.returncode = 1 + result.check_returncode = check_returncode + return result + + run_mock.side_effect = run_sideeffect + + with self.assertRaises(ValueError) as raised: + rust_watch.update_git_repo(pathlib.Path('/does/not/exist/at/all')) + + self.assertIs(raised.exception, oh_no_error) + self.assertEqual(run_call_count, 5) + + sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)] + self.assertEqual(sleep_mock.mock_calls, sleep_timings) + + @unittest.mock.patch.object(subprocess, 'run') + def test_get_new_gentoo_commits_functions(self, run_mock): + returned = unittest.mock.Mock() + returned.returncode = 0 + returned.stdout = '\n'.join(( + 'abc123 newer commit', + 'abcdef and an older commit', + )) + run_mock.return_value = returned + results = rust_watch.get_new_gentoo_commits( + pathlib.Path('/does/not/exist/at/all'), 'defabc') + self.assertEqual(results, [ + rust_watch.GitCommit('abcdef', 'and an older commit'), + rust_watch.GitCommit('abc123', 'newer commit'), + ]) + + def test_compose_email_on_a_new_release(self): + new_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[], + ) + + self.assertEqual(new_release, ('[rust-watch] new rustc release detected', + ['Rustc tag for v1.1.0 was found.'])) + + def test_compose_email_on_a_new_gentoo_commit(self): + sha_a = 'a' * 40 + new_commit = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) + + self.assertEqual(new_commit, + ('[rust-watch] new rust ebuild commit detected', [ + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]) + ])) + + def test_compose_email_on_multiple_events(self): + sha_a = 'a' * 40 + new_commit_and_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) + + self.assertEqual( + new_commit_and_release, + ('[rust-watch] new rustc release detected; new rust ebuild commit ' + 'detected', [ + 'Rustc tag for v1.1.0 was found.', + tiny_render.line_break, + tiny_render.line_break, + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]), + ])) + + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + +if __name__ == '__main__': + unittest.main() diff --git a/setup_chromeos.py b/setup_chromeos.py index 63206492..07c7530d 100755 --- a/setup_chromeos.py +++ b/setup_chromeos.py @@ -244,7 +244,7 @@ Default is 'latest_lkgm'.""") return cmd_executer.RunCommand( 'git ls-remote ' 'https://chrome-internal.googlesource.com/chrome/src-internal.git ' - '> /dev/null') + 'refs/HEAD > /dev/null') if __name__ == '__main__': diff --git a/toolchain_utils_githooks/check-presubmit.py b/toolchain_utils_githooks/check-presubmit.py index 26373342..274a75f4 100755 --- a/toolchain_utils_githooks/check-presubmit.py +++ b/toolchain_utils_githooks/check-presubmit.py @@ -7,11 +7,7 @@ """Runs presubmit checks against a bundle of files.""" -# To keep `cros lint` happy -from __future__ import division, print_function - import argparse -import collections import datetime import multiprocessing import multiprocessing.pool @@ -26,7 +22,9 @@ import traceback import typing as t -def run_command_unchecked(command, cwd, env=None): +def run_command_unchecked(command: t.List[str], + cwd: str, + env: t.Dict[str, str] = None) -> t.Tuple[int, str]: """Runs a command in the given dir, returning its exit code and stdio.""" p = subprocess.Popen( command, @@ -42,12 +40,12 @@ def run_command_unchecked(command, cwd, env=None): return exit_code, stdout.decode('utf-8', 'replace') -def has_executable_on_path(exe): +def has_executable_on_path(exe: str) -> bool: """Returns whether we have `exe` somewhere on our $PATH""" return shutil.which(exe) is not None -def escape_command(command): +def escape_command(command: t.Iterable[str]) -> str: """Returns a human-readable and copy-pastable shell command. Only intended for use in output to users. shell=True is strongly discouraged. @@ -55,18 +53,18 @@ def escape_command(command): return ' '.join(shlex.quote(x) for x in command) -def remove_deleted_files(files): +def remove_deleted_files(files: t.Iterable[str]) -> t.List[str]: return [f for f in files if os.path.exists(f)] -def is_file_executable(file_path): +def is_file_executable(file_path: str) -> bool: return os.access(file_path, os.X_OK) # As noted in our docs, some of our Python code depends on modules that sit in # toolchain-utils/. Add that to PYTHONPATH to ensure that things like `cros # lint` are kept happy. -def env_with_pythonpath(toolchain_utils_root): +def env_with_pythonpath(toolchain_utils_root: str) -> t.Dict[str, str]: env = dict(os.environ) if 'PYTHONPATH' in env: env['PYTHONPATH'] += ':' + toolchain_utils_root @@ -86,12 +84,22 @@ def env_with_pythonpath(toolchain_utils_root): # least ${number_of_concurrently_running_checkers}+1 threads are present # in the pool. In order words, blocking on results from the provided # threadpool is OK. -CheckResult = collections.namedtuple('CheckResult', - ('ok', 'output', 'autofix_commands')) +CheckResult = t.NamedTuple( + 'CheckResult', + ( + ('ok', bool), + ('output', str), + ('autofix_commands', t.List[t.List[str]]), + ), +) + +def get_check_result_or_catch( + task: multiprocessing.pool.ApplyResult) -> CheckResult: + """Returns the result of task(); if that raises, returns a CheckResult. -def get_check_result_or_catch(task): - """Returns the result of task(); if that raises, returns a CheckResult.""" + The task is expected to return a CheckResult on get(). + """ try: return task.get() except Exception: @@ -103,7 +111,8 @@ def get_check_result_or_catch(task): ) -def check_yapf(toolchain_utils_root, python_files): +def check_yapf(toolchain_utils_root: str, + python_files: t.Iterable[str]) -> CheckResult: """Subchecker of check_py_format. Checks python file formats with yapf""" command = ['yapf', '-d'] + python_files exit_code, stdout_and_stderr = run_command_unchecked( @@ -146,7 +155,7 @@ def check_yapf(toolchain_utils_root, python_files): ) -def check_python_file_headers(python_files): +def check_python_file_headers(python_files: t.Iterable[str]) -> CheckResult: """Subchecker of check_py_format. Checks python #!s""" add_hashbang = [] remove_hashbang = [] @@ -188,7 +197,9 @@ def check_python_file_headers(python_files): ) -def check_py_format(toolchain_utils_root, thread_pool, files): +def check_py_format(toolchain_utils_root: str, + thread_pool: multiprocessing.pool.ThreadPool, + files: t.Iterable[str]) -> CheckResult: """Runs yapf on files to check for style bugs. Also checks for #!s.""" yapf = 'yapf' if not has_executable_on_path(yapf): @@ -217,7 +228,13 @@ def check_py_format(toolchain_utils_root, thread_pool, files): return [(name, get_check_result_or_catch(task)) for name, task in tasks] -def check_cros_lint(toolchain_utils_root, thread_pool, files): +def find_chromeos_root_directory() -> t.Optional[str]: + return os.getenv('CHROMEOS_ROOT_DIRECTORY') + + +def check_cros_lint( + toolchain_utils_root: str, thread_pool: multiprocessing.pool.ThreadPool, + files: t.Iterable[str]) -> t.Union[t.List[CheckResult], CheckResult]: """Runs `cros lint`""" fixed_env = env_with_pythonpath(toolchain_utils_root) @@ -225,7 +242,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files): # We have to support users who don't have a chroot. So we either run `cros # lint` (if it's been made available to us), or we try a mix of # pylint+golint. - def try_run_cros_lint(cros_binary): + def try_run_cros_lint(cros_binary: str) -> t.Optional[CheckResult]: exit_code, output = run_command_unchecked( [cros_binary, 'lint', '--py3', '--'] + files, toolchain_utils_root, @@ -246,7 +263,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files): if cros_lint is not None: return cros_lint - cros_root = os.getenv('CHROMEOS_ROOT_DIRECTORY') + cros_root = find_chromeos_root_directory() if cros_root: cros_lint = try_run_cros_lint(os.path.join(cros_root, 'chromite/bin/cros')) if cros_lint is not None: @@ -254,7 +271,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files): tasks = [] - def check_result_from_command(command): + def check_result_from_command(command: t.List[str]) -> CheckResult: exit_code, output = run_command_unchecked( command, toolchain_utils_root, env=fixed_env) return CheckResult( @@ -266,7 +283,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files): python_files = [f for f in remove_deleted_files(files) if f.endswith('.py')] if python_files: - def run_pylint(): + def run_pylint() -> CheckResult: # pylint is required. Fail hard if it DNE. return check_result_from_command(['pylint'] + python_files) @@ -275,7 +292,7 @@ def check_cros_lint(toolchain_utils_root, thread_pool, files): go_files = [f for f in remove_deleted_files(files) if f.endswith('.go')] if go_files: - def run_golint(): + def run_golint() -> CheckResult: if has_executable_on_path('golint'): return check_result_from_command(['golint', '-set_exit_status'] + go_files) @@ -362,7 +379,9 @@ def check_go_format(toolchain_utils_root, _thread_pool, files): ) -def check_tests(toolchain_utils_root, _thread_pool, files): +def check_tests(toolchain_utils_root: str, + _thread_pool: multiprocessing.pool.ThreadPool, + files: t.List[str]) -> CheckResult: """Runs tests.""" exit_code, stdout_and_stderr = run_command_unchecked( [os.path.join(toolchain_utils_root, 'run_tests_for.py'), '--'] + files, @@ -374,15 +393,17 @@ def check_tests(toolchain_utils_root, _thread_pool, files): ) -def detect_toolchain_utils_root(): +def detect_toolchain_utils_root() -> str: return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -def process_check_result(check_name, check_results, start_time): +def process_check_result( + check_name: str, check_results: t.Union[t.List[CheckResult], CheckResult], + start_time: datetime.datetime) -> t.Tuple[bool, t.List[t.List[str]]]: """Prints human-readable output for the given check_results.""" indent = ' ' - def indent_block(text): + def indent_block(text: str) -> str: return indent + text.replace('\n', '\n' + indent) if isinstance(check_results, CheckResult): @@ -426,7 +447,8 @@ def process_check_result(check_name, check_results, start_time): return ok, autofix_commands -def try_autofix(all_autofix_commands, toolchain_utils_root): +def try_autofix(all_autofix_commands: t.List[t.List[str]], + toolchain_utils_root: str) -> None: """Tries to run all given autofix commands, if appropriate.""" if not all_autofix_commands: return @@ -478,10 +500,16 @@ def maybe_reexec_inside_chroot(autofix: bool, files: t.List[str]) -> None: return enter_chroot = True + chdir_to = None toolchain_utils = detect_toolchain_utils_root() if find_repo_root(toolchain_utils) is None: - print('Standalone toolchain-utils checkout detected; cannot enter chroot.') - enter_chroot = False + chromeos_root_dir = find_chromeos_root_directory() + if chromeos_root_dir is None: + print('Standalone toolchain-utils checkout detected; cannot enter ' + 'chroot.') + enter_chroot = False + else: + chdir_to = chromeos_root_dir if not has_executable_on_path('cros_sdk'): print('No `cros_sdk` detected on $PATH; cannot enter chroot.') @@ -511,7 +539,11 @@ def maybe_reexec_inside_chroot(autofix: bool, files: t.List[str]) -> None: args.append('--no_autofix') args.extend(rebase_path(x) for x in files) - print('Attempting to enter the chroot...') + if chdir_to is None: + print('Attempting to enter the chroot...') + else: + print(f'Attempting to enter the chroot for tree at {chdir_to}...') + os.chdir(chdir_to) os.execvp(args[0], args) @@ -534,7 +566,7 @@ def ensure_scipy_installed() -> None: subprocess.check_call(['pip', 'install', '--user', 'scipy']) -def main(argv): +def main(argv: t.List[str]) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--no_autofix', diff --git a/unblocked_terms.txt b/unblocked_terms.txt new file mode 100644 index 00000000..c961e667 --- /dev/null +++ b/unblocked_terms.txt @@ -0,0 +1,5 @@ +# Code uses this in many cases to refer to git branches. +master + +# A few paths in AOSP contain this term. +native |