From 2c5d62785d59e0cd7187d66c996294e42cb94deb Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Fri, 8 Oct 2021 18:28:28 +0000 Subject: compiler_wrapper: move llvmnext flag to post flags This CL moves flags added for LLVM Next that disabled variable_unused_but_set warnings to PostFlags in order to make sure it gets added after Wall and Wextra. BUG=b:195433889 TEST=reran package tests Change-Id: Icdd2cbe3ced7b8427257c87f9e527b0d0e8da57d Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3214640 Reviewed-by: Manoj Gupta Commit-Queue: Ryan Beltran Tested-by: Ryan Beltran --- compiler_wrapper/cros_llvm_next_flags.go | 7 ++++--- .../cros_hardened_llvmnext_golden/bisect.json | 6 +++--- .../cros_hardened_llvmnext_golden/clang_path.json | 24 +++++++++++----------- .../cros_hardened_llvmnext_golden/clangtidy.json | 16 +++++++-------- .../force_disable_werror.json | 10 ++++----- .../gcc_clang_syntax.json | 8 ++++---- 6 files changed, 36 insertions(+), 35 deletions(-) diff --git a/compiler_wrapper/cros_llvm_next_flags.go b/compiler_wrapper/cros_llvm_next_flags.go index d2ae413b..2211ae18 100644 --- a/compiler_wrapper/cros_llvm_next_flags.go +++ b/compiler_wrapper/cros_llvm_next_flags.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +//go:build !android_llvm_next_flags // +build !android_llvm_next_flags package main @@ -13,8 +14,8 @@ package main // tag is set. // TODO: Enable test in config_test.go, once we have new llvm-next flags. -var llvmNextFlags = []string{ +var llvmNextFlags = []string{} + +var llvmNextPostFlags = []string{ "-Wno-unused-but-set-variable", } - -var llvmNextPostFlags = []string{} diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index b29f09d0..995ce860 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -40,7 +40,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -55,6 +54,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -112,7 +112,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -127,6 +126,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -187,7 +187,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -202,6 +201,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json index 352ac4da..3e396ab9 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json @@ -30,7 +30,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -45,6 +44,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -93,7 +93,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -108,6 +107,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -156,7 +156,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -171,6 +170,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -219,7 +219,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -234,6 +233,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -289,7 +289,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -306,6 +305,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -364,7 +364,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -381,6 +380,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -434,7 +434,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -451,6 +450,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -499,7 +499,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -514,6 +513,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -559,7 +559,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -574,6 +573,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -619,7 +619,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -634,6 +633,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -679,7 +679,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -694,6 +693,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -742,7 +742,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -757,6 +756,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json index 11851288..0d4ee88f 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json @@ -45,7 +45,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -60,6 +59,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -86,7 +86,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -101,6 +100,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -157,7 +157,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -172,6 +171,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -199,7 +199,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -214,6 +213,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -272,7 +272,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -287,6 +286,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -317,7 +317,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -332,6 +331,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -391,7 +391,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -406,6 +405,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -433,7 +433,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -448,6 +447,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json index 44afa63d..fc55b4cd 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json @@ -33,7 +33,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -48,6 +47,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -96,7 +96,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -111,6 +110,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -145,7 +145,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -160,6 +159,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -212,7 +212,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -227,6 +226,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -261,7 +261,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -276,6 +275,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json index 2851f6d5..ede8dd00 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json @@ -30,7 +30,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -45,6 +44,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -116,7 +116,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -131,6 +130,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -197,7 +197,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -212,6 +211,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -260,7 +260,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -275,6 +274,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", -- cgit v1.2.3 From ec4d2aec19dc05d45eef5ea4925cbb72f2a2123f Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Fri, 8 Oct 2021 13:56:35 -0700 Subject: afdo_metadata: Publish the new kernel profiles Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 Update chromeos-kernel-5_4 BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator Change-Id: Ie0f2728c1a4800dbfe1f8f98ec98c73a6d8a33a6 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3214700 Reviewed-by: Bob Haarman Commit-Queue: Denis Nikitin Tested-by: Denis Nikitin --- afdo_metadata/kernel_afdo.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index a4044c67..b4d2687d 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,14 @@ { "chromeos-kernel-4_4": { - "name": "R96-14209.0-1632130382" + "name": "R96-14244.0-1633340299" }, "chromeos-kernel-4_14": { - "name": "R96-14221.0-1632130642" + "name": "R96-14244.0-1633340189" }, "chromeos-kernel-4_19": { - "name": "R96-14209.0-1632130511" + "name": "R96-14244.0-1633339943" }, "chromeos-kernel-5_4": { - "name": "R96-14209.0-1632130436" + "name": "R96-14244.0-1633339893" } } -- cgit v1.2.3 From c5e12fbac037054444b2c07d708745c377d68328 Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Thu, 7 Oct 2021 03:27:51 +0000 Subject: Add 'no_lock' crosperf setting This global experiment setting prevents crosperf from attempting to lock the associated devices. This is important if the devices were locked previously before by crosfleet. BUG=b:187535367 TEST=CQ TEST=Lease board from crosfleet. Forward SSH port. Run crosperf on forwarded port. Change-Id: I0f5341e07e325a613c6316778c0f096f6d8088a2 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3210525 Reviewed-by: Manoj Gupta Tested-by: Jordan R Abrahams Commit-Queue: Jordan R Abrahams --- crosperf/crosperf_unittest.py | 2 +- crosperf/experiment.py | 3 ++- crosperf/experiment_factory.py | 4 +++- crosperf/experiment_runner.py | 2 +- crosperf/settings_factory.py | 7 +++++++ crosperf/settings_factory_unittest.py | 4 ++-- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index 9c7d52a1..07728e7d 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -68,7 +68,7 @@ class CrosperfTest(unittest.TestCase): settings = crosperf.ConvertOptionsToSettings(options) self.assertIsNotNone(settings) self.assertIsInstance(settings, settings_factory.GlobalSettings) - self.assertEqual(len(settings.fields), 39) + self.assertEqual(len(settings.fields), 40) self.assertTrue(settings.GetField('rerun')) argv = ['crosperf/crosperf.py', 'temp.exp'] options, _ = parser.parse_known_args(argv) diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 854d7f77..e919f6ee 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -29,7 +29,7 @@ class Experiment(object): cache_conditions, labels, benchmarks, experiment_file, email_to, acquire_timeout, log_dir, log_level, share_cache, results_directory, compress_results, locks_directory, cwp_dso, - ignore_min_max, crosfleet, dut_config): + ignore_min_max, crosfleet, dut_config, no_lock: bool): self.name = name self.working_directory = working_directory self.remote = remote @@ -57,6 +57,7 @@ class Experiment(object): self.cwp_dso = cwp_dso self.ignore_min_max = ignore_min_max self.crosfleet = crosfleet + self.no_lock = no_lock self.l = logger.GetLogger(log_dir) if not self.benchmarks: diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 73928756..19b373a9 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -121,6 +121,7 @@ class ExperimentFactory(object): log_level = 'verbose' crosfleet = global_settings.GetField('crosfleet') + no_lock = bool(global_settings.GetField('no_lock')) # Check whether crosfleet tool is installed correctly for crosfleet mode. if crosfleet and not self.CheckCrosfleetTool(chromeos_root, log_level): sys.exit(0) @@ -444,7 +445,8 @@ class ExperimentFactory(object): experiment_file.Canonicalize(), email, acquire_timeout, log_dir, log_level, share_cache, results_dir, compress_results, locks_dir, cwp_dso, - ignore_min_max, crosfleet, dut_config) + ignore_min_max, crosfleet, dut_config, + no_lock=no_lock) return experiment diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 49aff425..ca846154 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -176,7 +176,7 @@ class ExperimentRunner(object): # no-op task on the DUT and new test created will be hanging there. # TODO(zhizhouy): Need to check whether machine is ready or not before # assigning a test to it. - if not experiment.crosfleet: + if not experiment.no_lock and not experiment.crosfleet: self._LockAllMachines(experiment) # Calculate all checksums of avaiable/locked machines, to ensure same # label has same machines for testing diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 34326b68..a38f24f9 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -439,6 +439,13 @@ class GlobalSettings(Settings): ' or equal to a percent of max_freq. ' 'CPU frequency is reduced to 95%% by default to reduce thermal ' 'throttling.')) + self.AddField( + BooleanField( + 'no_lock', + default=False, + description='Do not attempt to lock the DUT.' + ' Useful when lock is held externally, say with crosfleet.' + )) class SettingsFactory(object): diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index 035da8d7..8277e870 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -50,7 +50,7 @@ class GlobalSettingsTest(unittest.TestCase): def test_init(self): res = settings_factory.GlobalSettings('g_settings') self.assertIsNotNone(res) - self.assertEqual(len(res.fields), 39) + self.assertEqual(len(res.fields), 40) self.assertEqual(res.GetField('name'), '') self.assertEqual(res.GetField('board'), '') self.assertEqual(res.GetField('crosfleet'), False) @@ -108,7 +108,7 @@ class SettingsFactoryTest(unittest.TestCase): g_settings = settings_factory.SettingsFactory().GetSettings( 'global', 'global') self.assertIsInstance(g_settings, settings_factory.GlobalSettings) - self.assertEqual(len(g_settings.fields), 39) + self.assertEqual(len(g_settings.fields), 40) if __name__ == '__main__': -- cgit v1.2.3 From a7a19342ac9fe044cbf45a3aa618defca9db413f Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Sat, 9 Oct 2021 03:34:49 +0000 Subject: Auto-format the crosperf files Currently, these crosperf files do not abide by the pep8 standard that CrOS requires. This applies the necessary formatting to get past the commit hook. BUG=None TEST=CQ (formatting) Change-Id: I501fd37f8a1d20fd0b987587eb52eb582f18c1a1 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3215085 Tested-by: Jordan R Abrahams Reviewed-by: Manoj Gupta Commit-Queue: Jordan R Abrahams --- crosperf/crosperf_unittest.py | 13 +- crosperf/experiment_factory.py | 140 ++++++----- crosperf/experiment_runner.py | 36 +-- crosperf/settings_factory.py | 522 +++++++++++++++++++---------------------- 4 files changed, 340 insertions(+), 371 deletions(-) diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index 07728e7d..774159ff 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -55,13 +55,12 @@ class CrosperfTest(unittest.TestCase): def testConvertOptionsToSettings(self): parser = argparse.ArgumentParser() - parser.add_argument( - '-l', - '--log_dir', - dest='log_dir', - default='', - help='The log_dir, default is under ' - '/logs') + parser.add_argument('-l', + '--log_dir', + dest='log_dir', + default='', + help='The log_dir, default is under ' + '/logs') crosperf.SetupParserOptions(parser) argv = ['crosperf/crosperf.py', 'temp.exp', '--rerun=True'] options, _ = parser.parse_known_args(argv) diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 19b373a9..a9594a20 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -101,7 +101,8 @@ class ExperimentFactory(object): def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args, iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local, cwp_dso, weight): + show_all_results, retries, run_local, cwp_dso, + weight): """Add all the tests in a set to the benchmarks list.""" for test_name in benchmark_list: telemetry_benchmark = Benchmark(test_name, test_name, test_args, @@ -258,10 +259,10 @@ class ExperimentFactory(object): if suite == 'telemetry_Crosperf': if test_name == 'all_perfv2': - self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, test_args, - iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local, cwp_dso, - weight) + self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local, cwp_dso, weight) elif test_name == 'all_pagecyclers': self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests, test_args, iterations, rm_chroot_tmp, @@ -271,21 +272,20 @@ class ExperimentFactory(object): self.AppendBenchmarkSet(benchmarks, telemetry_crosbolt_perf_tests, test_args, iterations, rm_chroot_tmp, perf_args, 'telemetry_Crosperf', - show_all_results, retries, run_local, cwp_dso, - weight) - self.AppendBenchmarkSet( - benchmarks, - crosbolt_perf_tests, - '', - iterations, - rm_chroot_tmp, - perf_args, - '', - show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight) + show_all_results, retries, run_local, + cwp_dso, weight) + self.AppendBenchmarkSet(benchmarks, + crosbolt_perf_tests, + '', + iterations, + rm_chroot_tmp, + perf_args, + '', + show_all_results, + retries, + run_local=False, + cwp_dso=cwp_dso, + weight=weight) elif test_name == 'all_toolchain_perf': self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests, test_args, iterations, rm_chroot_tmp, @@ -325,10 +325,10 @@ class ExperimentFactory(object): # weight=weight)) elif test_name == 'all_toolchain_perf_old': self.AppendBenchmarkSet(benchmarks, - telemetry_toolchain_old_perf_tests, test_args, - iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local, cwp_dso, - weight) + telemetry_toolchain_old_perf_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local, cwp_dso, weight) else: benchmark = Benchmark(benchmark_name, test_name, test_args, iterations, rm_chroot_tmp, perf_args, suite, @@ -337,34 +337,32 @@ class ExperimentFactory(object): benchmarks.append(benchmark) else: if test_name == 'all_graphics_perf': - self.AppendBenchmarkSet( - benchmarks, - graphics_perf_tests, - '', - iterations, - rm_chroot_tmp, - perf_args, - '', - show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight) + self.AppendBenchmarkSet(benchmarks, + graphics_perf_tests, + '', + iterations, + rm_chroot_tmp, + perf_args, + '', + show_all_results, + retries, + run_local=False, + cwp_dso=cwp_dso, + weight=weight) else: # Add the single benchmark. - benchmark = Benchmark( - benchmark_name, - test_name, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight) + benchmark = Benchmark(benchmark_name, + test_name, + test_args, + iterations, + rm_chroot_tmp, + perf_args, + suite, + show_all_results, + retries, + run_local=False, + cwp_dso=cwp_dso, + weight=weight) benchmarks.append(benchmark) if not benchmarks: @@ -411,8 +409,8 @@ class ExperimentFactory(object): # TODO(yunlian): We should consolidate code in machine_manager.py # to derermine whether we are running from within google or not - if ('corp.google.com' in socket.gethostname() and not my_remote and - not crosfleet): + if ('corp.google.com' in socket.gethostname() and not my_remote + and not crosfleet): my_remote = self.GetDefaultRemotes(board) if global_settings.GetField('same_machine') and len(my_remote) > 1: raise RuntimeError('Only one remote is allowed when same_machine ' @@ -423,8 +421,8 @@ class ExperimentFactory(object): # pylint: disable=too-many-function-args label = MockLabel(label_name, build, image, autotest_path, debug_path, chromeos_root, board, my_remote, image_args, - cache_dir, cache_only, log_level, compiler, crosfleet, - chrome_src) + cache_dir, cache_only, log_level, compiler, + crosfleet, chrome_src) else: label = Label(label_name, build, image, autotest_path, debug_path, chromeos_root, board, my_remote, image_args, cache_dir, @@ -440,19 +438,33 @@ class ExperimentFactory(object): if crosfleet: for remote in all_remote: self.CheckRemotesInCrosfleet(remote) - experiment = Experiment(experiment_name, all_remote, working_directory, - chromeos_root, cache_conditions, labels, benchmarks, - experiment_file.Canonicalize(), email, - acquire_timeout, log_dir, log_level, share_cache, - results_dir, compress_results, locks_dir, cwp_dso, - ignore_min_max, crosfleet, dut_config, + experiment = Experiment(experiment_name, + all_remote, + working_directory, + chromeos_root, + cache_conditions, + labels, + benchmarks, + experiment_file.Canonicalize(), + email, + acquire_timeout, + log_dir, + log_level, + share_cache, + results_dir, + compress_results, + locks_dir, + cwp_dso, + ignore_min_max, + crosfleet, + dut_config, no_lock=no_lock) return experiment def GetDefaultRemotes(self, board): - default_remotes_file = os.path.join( - os.path.dirname(__file__), 'default_remotes') + default_remotes_file = os.path.join(os.path.dirname(__file__), + 'default_remotes') try: with open(default_remotes_file) as f: for line in f: @@ -482,8 +494,8 @@ class ExperimentFactory(object): l = logger.GetLogger() l.LogOutput('Crosfleet tool not installed, trying to install it.') ce = command_executer.GetCommandExecuter(l, log_level=log_level) - setup_lab_tools = os.path.join(chromeos_root, 'chromeos-admin', 'lab-tools', - 'setup_lab_tools') + setup_lab_tools = os.path.join(chromeos_root, 'chromeos-admin', + 'lab-tools', 'setup_lab_tools') cmd = '%s' % setup_lab_tools status = ce.RunCommand(cmd) if status != 0: diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index ca846154..6daef780 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -160,8 +160,8 @@ class ExperimentRunner(object): cache.Init(br.label.chromeos_image, br.label.chromeos_root, br.benchmark.test_name, br.iteration, br.test_args, br.profiler_args, br.machine_manager, br.machine, - br.label.board, br.cache_conditions, br.logger(), br.log_level, - br.label, br.share_cache, br.benchmark.suite, + br.label.board, br.cache_conditions, br.logger(), + br.log_level, br.label, br.share_cache, br.benchmark.suite, br.benchmark.show_all_results, br.benchmark.run_local, br.benchmark.cwp_dso) cache_dir = cache.GetCacheDirForWrite() @@ -236,8 +236,8 @@ class ExperimentRunner(object): if not benchmark_run.cache_hit: send_mail = True break - if (not send_mail and not experiment.email_to or - config.GetConfig('no_email')): + if (not send_mail and not experiment.email_to + or config.GetConfig('no_email')): return label_names = [] @@ -245,7 +245,8 @@ class ExperimentRunner(object): label_names.append(label.name) subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names)) - text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() + text_report = TextResultsReport.FromExperiment(experiment, + True).GetReport() text_report += ('\nResults are stored in %s.\n' % experiment.results_directory) text_report = "
%s
" % text_report @@ -253,12 +254,11 @@ class ExperimentRunner(object): attachment = EmailSender.Attachment('report.html', html_report) email_to = experiment.email_to or [] email_to.append(getpass.getuser()) - EmailSender().SendEmail( - email_to, - subject, - text_report, - attachments=[attachment], - msg_type='html') + EmailSender().SendEmail(email_to, + subject, + text_report, + attachments=[attachment], + msg_type='html') def _StoreResults(self, experiment): if self._terminated: @@ -300,9 +300,10 @@ class ExperimentRunner(object): self.l.LogOutput('Storing results of each benchmark run.') for benchmark_run in experiment.benchmark_runs: if benchmark_run.result: - benchmark_run_name = ''.join( - ch for ch in benchmark_run.name if ch.isalnum()) - benchmark_run_path = os.path.join(results_directory, benchmark_run_name) + benchmark_run_name = ''.join(ch for ch in benchmark_run.name + if ch.isalnum()) + benchmark_run_path = os.path.join(results_directory, + benchmark_run_name) if experiment.compress_results: benchmark_run.result.CompressResultsTo(benchmark_run_path) else: @@ -313,15 +314,16 @@ class ExperimentRunner(object): results_table_path = os.path.join(results_directory, 'results.html') report = HTMLResultsReport.FromExperiment(experiment).GetReport() if self.json_report: - json_report = JSONResultsReport.FromExperiment( - experiment, json_args={'indent': 2}) + json_report = JSONResultsReport.FromExperiment(experiment, + json_args={'indent': 2}) _WriteJSONReportToFile(experiment, results_directory, json_report) FileUtils().WriteFile(results_table_path, report) self.l.LogOutput('Storing email message body in %s.' % results_directory) msg_file_path = os.path.join(results_directory, 'msg_body.html') - text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() + text_report = TextResultsReport.FromExperiment(experiment, + True).GetReport() text_report += ('\nResults are stored in %s.\n' % experiment.results_directory) msg_body = "
%s
" % text_report diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index a38f24f9..78834c63 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -22,14 +22,13 @@ class BenchmarkSettings(Settings): def __init__(self, name): super(BenchmarkSettings, self).__init__(name, 'benchmark') self.AddField( - TextField( - 'test_name', - description='The name of the test to run. ' - 'Defaults to the name of the benchmark.')) + TextField('test_name', + description='The name of the test to run. ' + 'Defaults to the name of the benchmark.')) self.AddField( - TextField( - 'test_args', description='Arguments to be passed to the ' - 'test.')) + TextField('test_args', + description='Arguments to be passed to the ' + 'test.')) self.AddField( IntegerField( 'iterations', @@ -39,24 +38,21 @@ class BenchmarkSettings(Settings): 'If not set, will run each benchmark test the optimum number of ' 'times to get a stable result.')) self.AddField( - TextField( - 'suite', - default='test_that', - description='The type of the benchmark.')) + TextField('suite', + default='test_that', + description='The type of the benchmark.')) self.AddField( - IntegerField( - 'retries', - default=0, - description='Number of times to retry a ' - 'benchmark run.')) + IntegerField('retries', + default=0, + description='Number of times to retry a ' + 'benchmark run.')) self.AddField( - BooleanField( - 'run_local', - description='Run benchmark harness on the DUT. ' - 'Currently only compatible with the suite: ' - 'telemetry_Crosperf.', - required=False, - default=True)) + BooleanField('run_local', + description='Run benchmark harness on the DUT. ' + 'Currently only compatible with the suite: ' + 'telemetry_Crosperf.', + required=False, + default=True)) self.AddField( FloatField( 'weight', @@ -70,12 +66,11 @@ class LabelSettings(Settings): def __init__(self, name): super(LabelSettings, self).__init__(name, 'label') self.AddField( - TextField( - 'chromeos_image', - required=False, - description='The path to the image to run tests ' - 'on, for local/custom-built images. See the ' - "'build' option for official or trybot images.")) + TextField('chromeos_image', + required=False, + description='The path to the image to run tests ' + 'on, for local/custom-built images. See the ' + "'build' option for official or trybot images.")) self.AddField( TextField( 'autotest_path', @@ -90,53 +85,46 @@ class LabelSettings(Settings): description='Debug info directory relative to chroot which has ' 'symbols and vmlinux that can be used by perf tool.')) self.AddField( - TextField( - 'chromeos_root', - description='The path to a chromeos checkout which ' - 'contains a src/scripts directory. Defaults to ' - 'the chromeos checkout which contains the ' - 'chromeos_image.')) - self.AddField( - ListField( - 'remote', - description='A comma-separated list of IPs of chromeos' - 'devices to run experiments on.')) - self.AddField( - TextField( - 'image_args', - required=False, - default='', - description='Extra arguments to pass to ' - 'image_chromeos.py.')) - self.AddField( - TextField( - 'cache_dir', - default='', - description='The cache dir for this image.')) - self.AddField( - TextField( - 'compiler', - default='gcc', - description='The compiler used to build the ' - 'ChromeOS image (gcc or llvm).')) - self.AddField( - TextField( - 'chrome_src', - description='The path to the source of chrome. ' - 'This is used to run telemetry benchmarks. ' - 'The default one is the src inside chroot.', - required=False, - default='')) - self.AddField( - TextField( - 'build', - description='The xbuddy specification for an ' - 'official or trybot image to use for tests. ' - "'/remote' is assumed, and the board is given " - "elsewhere, so omit the '/remote//' xbuddy " - 'prefix.', - required=False, - default='')) + TextField('chromeos_root', + description='The path to a chromeos checkout which ' + 'contains a src/scripts directory. Defaults to ' + 'the chromeos checkout which contains the ' + 'chromeos_image.')) + self.AddField( + ListField('remote', + description='A comma-separated list of IPs of chromeos' + 'devices to run experiments on.')) + self.AddField( + TextField('image_args', + required=False, + default='', + description='Extra arguments to pass to ' + 'image_chromeos.py.')) + self.AddField( + TextField('cache_dir', + default='', + description='The cache dir for this image.')) + self.AddField( + TextField('compiler', + default='gcc', + description='The compiler used to build the ' + 'ChromeOS image (gcc or llvm).')) + self.AddField( + TextField('chrome_src', + description='The path to the source of chrome. ' + 'This is used to run telemetry benchmarks. ' + 'The default one is the src inside chroot.', + required=False, + default='')) + self.AddField( + TextField('build', + description='The xbuddy specification for an ' + 'official or trybot image to use for tests. ' + "'/remote' is assumed, and the board is given " + "elsewhere, so omit the '/remote//' xbuddy " + 'prefix.', + required=False, + default='')) class GlobalSettings(Settings): @@ -145,67 +133,56 @@ class GlobalSettings(Settings): def __init__(self, name): super(GlobalSettings, self).__init__(name, 'global') self.AddField( - TextField( - 'name', - description='The name of the experiment. Just an ' - 'identifier.')) - self.AddField( - TextField( - 'board', - description='The target board for running ' - 'experiments on, e.g. x86-alex.')) - self.AddField( - BooleanField( - 'crosfleet', - description='Whether to run experiments via crosfleet.', - default=False)) - self.AddField( - ListField( - 'remote', - description='A comma-separated list of IPs of ' - 'chromeos devices to run experiments on.')) - self.AddField( - BooleanField( - 'rerun_if_failed', - description='Whether to re-run failed test runs ' - 'or not.', - default=False)) - self.AddField( - BooleanField( - 'rm_chroot_tmp', - default=False, - description='Whether to remove the test_that ' - 'result in the chroot.')) - self.AddField( - ListField( - 'email', - description='Space-separated list of email ' - 'addresses to send email to.')) - self.AddField( - BooleanField( - 'rerun', - description='Whether to ignore the cache and ' - 'for tests to be re-run.', - default=False)) - self.AddField( - BooleanField( - 'same_specs', - default=True, - description='Ensure cached runs are run on the ' - 'same kind of devices which are specified as a ' - 'remote.')) - self.AddField( - BooleanField( - 'same_machine', - default=False, - description='Ensure cached runs are run on the ' - 'same remote.')) - self.AddField( - BooleanField( - 'use_file_locks', - default=False, - description='DEPRECATED: Whether to use the file locks ' - 'or AFE server lock mechanism.')) + TextField('name', + description='The name of the experiment. Just an ' + 'identifier.')) + self.AddField( + TextField('board', + description='The target board for running ' + 'experiments on, e.g. x86-alex.')) + self.AddField( + BooleanField('crosfleet', + description='Whether to run experiments via crosfleet.', + default=False)) + self.AddField( + ListField('remote', + description='A comma-separated list of IPs of ' + 'chromeos devices to run experiments on.')) + self.AddField( + BooleanField('rerun_if_failed', + description='Whether to re-run failed test runs ' + 'or not.', + default=False)) + self.AddField( + BooleanField('rm_chroot_tmp', + default=False, + description='Whether to remove the test_that ' + 'result in the chroot.')) + self.AddField( + ListField('email', + description='Space-separated list of email ' + 'addresses to send email to.')) + self.AddField( + BooleanField('rerun', + description='Whether to ignore the cache and ' + 'for tests to be re-run.', + default=False)) + self.AddField( + BooleanField('same_specs', + default=True, + description='Ensure cached runs are run on the ' + 'same kind of devices which are specified as a ' + 'remote.')) + self.AddField( + BooleanField('same_machine', + default=False, + description='Ensure cached runs are run on the ' + 'same remote.')) + self.AddField( + BooleanField('use_file_locks', + default=False, + description='DEPRECATED: Whether to use the file locks ' + 'or AFE server lock mechanism.')) self.AddField( IntegerField( 'iterations', @@ -215,79 +192,68 @@ class GlobalSettings(Settings): 'If not set, will run each benchmark test the optimum number of ' 'times to get a stable result.')) self.AddField( - TextField( - 'chromeos_root', - description='The path to a chromeos checkout which ' - 'contains a src/scripts directory. Defaults to ' - 'the chromeos checkout which contains the ' - 'chromeos_image.')) - self.AddField( - TextField( - 'logging_level', - default='average', - description='The level of logging desired. ' - "Options are 'quiet', 'average', and 'verbose'.")) - self.AddField( - IntegerField( - 'acquire_timeout', - default=0, - description='Number of seconds to wait for ' - 'machine before exit if all the machines in ' - 'the experiment file are busy. Default is 0.')) - self.AddField( - TextField( - 'perf_args', - default='', - description='The optional profile command. It ' - 'enables perf commands to record perforamance ' - 'related counters. It must start with perf ' - 'command record or stat followed by arguments.')) - self.AddField( - BooleanField( - 'download_debug', - default=True, - description='Download compressed debug symbols alongwith ' - 'image. This can provide more info matching symbols for' - 'profiles, but takes larger space. By default, download' - 'it only when perf_args is specified.')) - self.AddField( - TextField( - 'cache_dir', - default='', - description='The abs path of cache dir. ' - 'Default is /home/$(whoami)/cros_scratch.')) - self.AddField( - BooleanField( - 'cache_only', - default=False, - description='Whether to use only cached ' - 'results (do not rerun failed tests).')) - self.AddField( - BooleanField( - 'no_email', - default=False, - description='Whether to disable the email to ' - 'user after crosperf finishes.')) - self.AddField( - BooleanField( - 'json_report', - default=False, - description='Whether to generate a json version ' - 'of the report, for archiving.')) - self.AddField( - BooleanField( - 'show_all_results', - default=False, - description='When running Telemetry tests, ' - 'whether to all the results, instead of just ' - 'the default (summary) results.')) - self.AddField( - TextField( - 'share_cache', - default='', - description='Path to alternate cache whose data ' - 'you want to use. It accepts multiple directories ' - 'separated by a ",".')) + TextField('chromeos_root', + description='The path to a chromeos checkout which ' + 'contains a src/scripts directory. Defaults to ' + 'the chromeos checkout which contains the ' + 'chromeos_image.')) + self.AddField( + TextField('logging_level', + default='average', + description='The level of logging desired. ' + "Options are 'quiet', 'average', and 'verbose'.")) + self.AddField( + IntegerField('acquire_timeout', + default=0, + description='Number of seconds to wait for ' + 'machine before exit if all the machines in ' + 'the experiment file are busy. Default is 0.')) + self.AddField( + TextField('perf_args', + default='', + description='The optional profile command. It ' + 'enables perf commands to record perforamance ' + 'related counters. It must start with perf ' + 'command record or stat followed by arguments.')) + self.AddField( + BooleanField('download_debug', + default=True, + description='Download compressed debug symbols alongwith ' + 'image. This can provide more info matching symbols for' + 'profiles, but takes larger space. By default, download' + 'it only when perf_args is specified.')) + self.AddField( + TextField('cache_dir', + default='', + description='The abs path of cache dir. ' + 'Default is /home/$(whoami)/cros_scratch.')) + self.AddField( + BooleanField('cache_only', + default=False, + description='Whether to use only cached ' + 'results (do not rerun failed tests).')) + self.AddField( + BooleanField('no_email', + default=False, + description='Whether to disable the email to ' + 'user after crosperf finishes.')) + self.AddField( + BooleanField('json_report', + default=False, + description='Whether to generate a json version ' + 'of the report, for archiving.')) + self.AddField( + BooleanField('show_all_results', + default=False, + description='When running Telemetry tests, ' + 'whether to all the results, instead of just ' + 'the default (summary) results.')) + self.AddField( + TextField('share_cache', + default='', + description='Path to alternate cache whose data ' + 'you want to use. It accepts multiple directories ' + 'separated by a ",".')) self.AddField( TextField('results_dir', default='', description='The results dir.')) self.AddField( @@ -297,55 +263,49 @@ class GlobalSettings(Settings): description='Whether to compress all test results other than ' 'reports into a tarball to save disk space.')) self.AddField( - TextField( - 'locks_dir', - default='', - description='An alternate directory to use for ' - 'storing/checking machine file locks for local machines. ' - 'By default the file locks directory is ' - '/google/data/rw/users/mo/mobiletc-prebuild/locks.\n' - 'WARNING: If you use your own locks directory, ' - 'there is no guarantee that someone else might not ' - 'hold a lock on the same machine in a different ' - 'locks directory.')) - self.AddField( - TextField( - 'chrome_src', - description='The path to the source of chrome. ' - 'This is used to run telemetry benchmarks. ' - 'The default one is the src inside chroot.', - required=False, - default='')) - self.AddField( - IntegerField( - 'retries', - default=0, - description='Number of times to retry a ' - 'benchmark run.')) - self.AddField( - TextField( - 'cwp_dso', - description='The DSO type that we want to use for ' - 'CWP approximation. This is used to run telemetry ' - 'benchmarks. Valid DSO types can be found from dso_list ' - 'in experiment_factory.py. The default value is set to ' - 'be empty.', - required=False, - default='')) - self.AddField( - BooleanField( - 'enable_aslr', - description='Enable ASLR on the machine to run the ' - 'benchmarks. ASLR is disabled by default', - required=False, - default=False)) - self.AddField( - BooleanField( - 'ignore_min_max', - description='When doing math for the raw results, ' - 'ignore min and max values to reduce noise.', - required=False, - default=False)) + TextField('locks_dir', + default='', + description='An alternate directory to use for ' + 'storing/checking machine file locks for local machines. ' + 'By default the file locks directory is ' + '/google/data/rw/users/mo/mobiletc-prebuild/locks.\n' + 'WARNING: If you use your own locks directory, ' + 'there is no guarantee that someone else might not ' + 'hold a lock on the same machine in a different ' + 'locks directory.')) + self.AddField( + TextField('chrome_src', + description='The path to the source of chrome. ' + 'This is used to run telemetry benchmarks. ' + 'The default one is the src inside chroot.', + required=False, + default='')) + self.AddField( + IntegerField('retries', + default=0, + description='Number of times to retry a ' + 'benchmark run.')) + self.AddField( + TextField('cwp_dso', + description='The DSO type that we want to use for ' + 'CWP approximation. This is used to run telemetry ' + 'benchmarks. Valid DSO types can be found from dso_list ' + 'in experiment_factory.py. The default value is set to ' + 'be empty.', + required=False, + default='')) + self.AddField( + BooleanField('enable_aslr', + description='Enable ASLR on the machine to run the ' + 'benchmarks. ASLR is disabled by default', + required=False, + default=False)) + self.AddField( + BooleanField('ignore_min_max', + description='When doing math for the raw results, ' + 'ignore min and max values to reduce noise.', + required=False, + default=False)) self.AddField( TextField( 'intel_pstate', @@ -356,12 +316,11 @@ class GlobalSettings(Settings): required=False, default='no_hwp')) self.AddField( - BooleanField( - 'turbostat', - description='Run turbostat process in the background' - ' of a benchmark. Enabled by default.', - required=False, - default=True)) + BooleanField('turbostat', + description='Run turbostat process in the background' + ' of a benchmark. Enabled by default.', + required=False, + default=True)) self.AddField( FloatField( 'top_interval', @@ -377,22 +336,20 @@ class GlobalSettings(Settings): required=False, default=1)) self.AddField( - IntegerField( - 'cooldown_temp', - required=False, - default=40, - description='Wait until CPU temperature goes down below' - ' specified temperature in Celsius' - ' prior starting a benchmark. ' - 'By default the value is set to 40 degrees.')) - self.AddField( - IntegerField( - 'cooldown_time', - required=False, - default=10, - description='Wait specified time in minutes allowing' - ' CPU to cool down. Zero value disables cooldown. ' - 'The default value is 10 minutes.')) + IntegerField('cooldown_temp', + required=False, + default=40, + description='Wait until CPU temperature goes down below' + ' specified temperature in Celsius' + ' prior starting a benchmark. ' + 'By default the value is set to 40 degrees.')) + self.AddField( + IntegerField('cooldown_time', + required=False, + default=10, + description='Wait specified time in minutes allowing' + ' CPU to cool down. Zero value disables cooldown. ' + 'The default value is 10 minutes.')) self.AddField( EnumField( 'governor', @@ -444,8 +401,7 @@ class GlobalSettings(Settings): 'no_lock', default=False, description='Do not attempt to lock the DUT.' - ' Useful when lock is held externally, say with crosfleet.' - )) + ' Useful when lock is held externally, say with crosfleet.')) class SettingsFactory(object): -- cgit v1.2.3 From dcb168bf3e794bef2cbd3f4307d3e29c046da9ef Mon Sep 17 00:00:00 2001 From: Manoj Gupta Date: Sun, 17 Oct 2021 14:06:17 -0700 Subject: afdo_metadata: Publish the new kernel profiles Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 Update chromeos-kernel-5_4 BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator Change-Id: Ia538fab15c409b67bfe21de04db3dc3a1e4ab544 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3228899 Tested-by: Manoj Gupta Auto-Submit: Manoj Gupta Commit-Queue: Denis Nikitin Reviewed-by: Denis Nikitin --- afdo_metadata/kernel_afdo.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index b4d2687d..982770f9 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,14 @@ { "chromeos-kernel-4_4": { - "name": "R96-14244.0-1633340299" + "name": "R97-14263.0-1633944739" }, "chromeos-kernel-4_14": { - "name": "R96-14244.0-1633340189" + "name": "R97-14263.0-1633944973" }, "chromeos-kernel-4_19": { - "name": "R96-14244.0-1633339943" + "name": "R97-14263.0-1633944768" }, "chromeos-kernel-5_4": { - "name": "R96-14244.0-1633339893" + "name": "R97-14263.0-1633944667" } } -- cgit v1.2.3 From 217c68a15b816dca4e11a21d2365386fd05aa62b Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Wed, 13 Oct 2021 21:41:45 +0000 Subject: Add crosperf_autolock.py wrapper This crosperf wrapper allows the user to lock devices by board type. It will query the swarm using the crosfleet API to find and lock an appropriate board. The remaining arguments are passed to crosperf, so the command will act like crosperf otherwise, once the remote has been set. One limitation is that this wrapper does not conduct any SSH port forwarding (which would be required for flashing devices from a cloudtop, for example). BUG=b:187535367 TEST=crosperf_autolock.py --board chell chell_exp.exp Change-Id: If8dc4630c42be63d6427716654e7dd7fa4132df5 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3221950 Reviewed-by: Manoj Gupta Reviewed-by: George Burgess Commit-Queue: Jordan R Abrahams Tested-by: Jordan R Abrahams --- crosperf/crosperf_autolock.py | 281 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100755 crosperf/crosperf_autolock.py diff --git a/crosperf/crosperf_autolock.py b/crosperf/crosperf_autolock.py new file mode 100755 index 00000000..b593fa9c --- /dev/null +++ b/crosperf/crosperf_autolock.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 + +# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Wrapper script to automatically lock devices for crosperf.""" + +import os +import sys +import argparse +import subprocess +import contextlib +import json +from typing import Optional, Any +import dataclasses + +# Have to do sys.path hackery because crosperf relies on PYTHONPATH +# modifications. +PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PARENT_DIR) + + +def main(sys_args: list[str]) -> Optional[str]: + """Run crosperf_autolock. Returns error msg or None""" + args, leftover_args = parse_args(sys_args) + fleet_params = [ + CrosfleetParams(board=args.board, + pool=args.pool, + lease_time=args.lease_time) + for _ in range(args.num_leases) + ] + if not fleet_params: + return ('No board names identified. If you want to use' + ' a known host, just use crosperf directly.') + try: + _run_crosperf(fleet_params, args.dut_lock_timeout, leftover_args) + except BoardLockError as e: + _eprint('ERROR:', e) + _eprint('May need to login to crosfleet? Run "crosfleet login"') + _eprint('The leases may also be successful later on. ' + 'Check with "crosfleet dut leases"') + return 'crosperf_autolock failed' + except BoardReleaseError as e: + _eprint('ERROR:', e) + _eprint('May need to re-run "crosfleet dut abandon"') + return 'crosperf_autolock failed' + return None + + +def parse_args(args: list[str]) -> tuple[Any, list]: + """Parse the CLI arguments.""" + parser = argparse.ArgumentParser( + 'crosperf_autolock', + description='Wrapper around crosperf' + ' to autolock DUTs from crosfleet.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--board', + type=str, + help='Space or comma separated list of boards to lock', + required=True, + default=argparse.SUPPRESS) + parser.add_argument('--num-leases', + type=int, + help='Number of boards to lock.', + metavar='NUM', + default=1) + parser.add_argument('--pool', + type=str, + help='Pool to pull from.', + default='DUT_POOL_QUOTA') + parser.add_argument('--dut-lock-timeout', + type=float, + metavar='SEC', + help='Number of seconds we want to try to lease a board' + ' from crosfleet. This option does NOT change the' + ' lease length.', + default=600) + parser.add_argument('--lease-time', + type=int, + metavar='MIN', + help='Number of minutes to lock the board. Max is 1440.', + default=1440) + parser.epilog = ( + 'For more detailed flags, you have to read the args taken by the' + ' crosperf executable. Args are passed transparently to crosperf.') + return parser.parse_known_args(args) + + +class BoardLockError(Exception): + """Error to indicate failure to lock a board.""" + + def __init__(self, msg: str): + self.msg = 'BoardLockError: ' + msg + super().__init__(self.msg) + + +class BoardReleaseError(Exception): + """Error to indicate failure to release a board.""" + + def __init__(self, msg: str): + self.msg = 'BoardReleaseError: ' + msg + super().__init__(self.msg) + + +@dataclasses.dataclass(frozen=True) +class CrosfleetParams: + """Dataclass to hold all crosfleet parameterizations.""" + board: str + pool: str + lease_time: int + + +def _eprint(*msg, **kwargs): + print(*msg, file=sys.stderr, **kwargs) + + +def _run_crosperf(crosfleet_params: list[CrosfleetParams], lock_timeout: float, + leftover_args: list[str]): + """Autolock devices and run crosperf with leftover arguments. + + Raises: + BoardLockError: When board was unable to be locked. + BoardReleaseError: When board was unable to be released. + """ + if not crosfleet_params: + raise ValueError('No crosfleet params given; cannot call crosfleet.') + + # We'll assume all the boards are the same type, which seems to be the case + # in experiments that actually get used. + passed_board_arg = crosfleet_params[0].board + with contextlib.ExitStack() as stack: + dut_hostnames = [] + for param in crosfleet_params: + print( + f'Sent lock request for {param.board} for {param.lease_time} minutes' + '\nIf this fails, you may need to run "crosfleet dut abandon <...>"') + # May raise BoardLockError, abandoning previous DUTs. + dut_hostname = stack.enter_context( + crosfleet_machine_ctx( + param.board, + param.lease_time, + lock_timeout, + {'label-pool': param.pool}, + )) + if dut_hostname: + print(f'Locked {param.board} machine: {dut_hostname}') + dut_hostnames.append(dut_hostname) + + # We import crosperf late, because this import is extremely slow. + # We don't want the user to wait several seconds just to get + # help info. + import crosperf + for dut_hostname in dut_hostnames: + crosperf.Main([ + sys.argv[0], + '--no_lock', + 'True', + '--remote', + dut_hostname, + '--board', + passed_board_arg, + ] + leftover_args) + + +@contextlib.contextmanager +def crosfleet_machine_ctx(board: str, + lease_minutes: int, + lock_timeout: float, + dims: dict[str, Any], + abandon_timeout: float = 120.0) -> Any: + """Acquire dut from crosfleet, and release once it leaves the context. + + Args: + board: Board type to lease. + lease_minutes: Length of lease, in minutes. + lock_timeout: How long to wait for a lock until quitting. + dims: Dictionary of dimension arguments to pass to crosfleet's '-dims' + abandon_timeout (optional): How long to wait for releasing until quitting. + + Yields: + A string representing the crosfleet DUT hostname. + + Raises: + BoardLockError: When board was unable to be locked. + BoardReleaseError: When board was unable to be released. + """ + # This lock may raise an exception, but if it does, we can't release + # the DUT anyways as we won't have the dut_hostname. + dut_hostname = crosfleet_autolock(board, lease_minutes, dims, lock_timeout) + try: + yield dut_hostname + finally: + if dut_hostname: + crosfleet_release(dut_hostname, abandon_timeout) + + +def crosfleet_autolock(board: str, lease_minutes: int, dims: dict[str, Any], + timeout_sec: float) -> str: + """Lock a device using crosfleet, paramaterized by the board type. + + Args: + board: Board of the DUT we want to lock. + lease_minutes: Number of minutes we're trying to lease the DUT for. + dims: Dictionary of dimension arguments to pass to crosfleet's '-dims' + timeout_sec: Number of seconds to try to lease the DUT. Default 120s. + + Returns: + The hostname of the board, or empty string if it couldn't be parsed. + + Raises: + BoardLockError: When board was unable to be locked. + """ + crosfleet_cmd_args = [ + 'crosfleet', + 'dut', + 'lease', + '-json', + '-reason="crosperf autolock"', + f'-board={board}', + f'-minutes={lease_minutes}', + ] + if dims: + dims_arg = ','.join('{}={}'.format(k, v) for k, v in dims.items()) + crosfleet_cmd_args.extend(['-dims', f'{dims_arg}']) + + try: + output = subprocess.check_output(crosfleet_cmd_args, + timeout=timeout_sec, + encoding='utf-8') + except subprocess.CalledProcessError as e: + raise BoardLockError( + f'crosfleet dut lease failed with exit code: {e.returncode}') + except subprocess.TimeoutExpired as e: + raise BoardLockError(f'crosfleet dut lease timed out after {timeout_sec}s;' + ' please abandon the dut manually.') + + try: + json_obj = json.loads(output) + dut_hostname = json_obj['DUT']['Hostname'] + if not isinstance(dut_hostname, str): + raise TypeError('dut_hostname was not a string') + except (json.JSONDecodeError, IndexError, KeyError, TypeError) as e: + raise BoardLockError( + f'crosfleet dut lease output was parsed incorrectly: {e!r};' + f' observed output was {output}') + return _maybe_append_suffix(dut_hostname) + + +def crosfleet_release(dut_hostname: str, timeout_sec: float = 120.0): + """Release a crosfleet device. + + Consider using the context managed crosfleet_machine_context + + Args: + dut_hostname: Name of the device we want to release. + timeout_sec: Number of seconds to try to release the DUT. Default is 120s. + + Raises: + BoardReleaseError: Potentially failed to abandon the lease. + """ + crosfleet_cmd_args = [ + 'crosfleet', + 'dut', + 'abandon', + dut_hostname, + ] + exit_code = subprocess.call(crosfleet_cmd_args, timeout=timeout_sec) + if exit_code != 0: + raise BoardReleaseError( + f'"crosfleet dut abandon" had exit code {exit_code}') + + +def _maybe_append_suffix(hostname: str) -> str: + if hostname.endswith('.cros') or '.cros.' in hostname: + return hostname + return hostname + '.cros' + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) -- cgit v1.2.3 From afd2d4b05fac5f3a71fd78e251f04cc54cb65c57 Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Wed, 20 Oct 2021 21:48:38 +0000 Subject: compiler_wrapper: ignore unused_but_set_variable This CL adds -Wno-unused-but-set-variable to clang post flags to prevent these warnings from blocking the LLVM uprev. BUG=b:195433889 TEST=cq Change-Id: I5195412183e2684aecd20b6ff2c3345f8b8b9dfb Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3235575 Reviewed-by: Manoj Gupta Commit-Queue: Ryan Beltran Tested-by: Ryan Beltran --- compiler_wrapper/config.go | 2 ++ compiler_wrapper/cros_llvm_next_flags.go | 4 +--- compiler_wrapper/testdata/cros_hardened_golden/bisect.json | 3 +++ .../clang_ftrapv_maincc_target_specific.json | 9 +++++++++ .../cros_hardened_golden/clang_maincc_target_specific.json | 9 +++++++++ .../testdata/cros_hardened_golden/clang_path.json | 12 ++++++++++++ .../testdata/cros_hardened_golden/clang_sanitizer_args.json | 8 ++++++++ .../testdata/cros_hardened_golden/clang_specific_args.json | 4 ++++ .../cros_hardened_golden/clang_sysroot_wrapper_common.json | 6 ++++++ .../testdata/cros_hardened_golden/clangtidy.json | 8 ++++++++ .../testdata/cros_hardened_golden/force_disable_werror.json | 5 +++++ .../testdata/cros_hardened_golden/gcc_clang_syntax.json | 4 ++++ .../testdata/cros_hardened_noccache_golden/bisect.json | 3 +++ .../testdata/cros_hardened_noccache_golden/clang_path.json | 12 ++++++++++++ .../testdata/cros_hardened_noccache_golden/clangtidy.json | 8 ++++++++ .../cros_hardened_noccache_golden/force_disable_werror.json | 5 +++++ .../cros_hardened_noccache_golden/gcc_clang_syntax.json | 4 ++++ .../testdata/cros_nonhardened_golden/bisect.json | 3 +++ .../clang_ftrapv_maincc_target_specific.json | 9 +++++++++ .../clang_maincc_target_specific.json | 9 +++++++++ .../testdata/cros_nonhardened_golden/clang_path.json | 12 ++++++++++++ .../cros_nonhardened_golden/clang_sanitizer_args.json | 8 ++++++++ .../cros_nonhardened_golden/clang_specific_args.json | 4 ++++ .../clang_sysroot_wrapper_common.json | 6 ++++++ .../testdata/cros_nonhardened_golden/clangtidy.json | 8 ++++++++ .../cros_nonhardened_golden/force_disable_werror.json | 5 +++++ .../testdata/cros_nonhardened_golden/gcc_clang_syntax.json | 4 ++++ 27 files changed, 171 insertions(+), 3 deletions(-) diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go index 11d77ba7..409456f8 100644 --- a/compiler_wrapper/config.go +++ b/compiler_wrapper/config.go @@ -154,6 +154,7 @@ var crosHardenedConfig = &config{ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", }, newWarningsDir: "/tmp/fatal_clang_warnings", triciumNitsDir: "/tmp/linting_output/clang-tidy", @@ -195,6 +196,7 @@ var crosNonHardenedConfig = &config{ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", }, newWarningsDir: "/tmp/fatal_clang_warnings", triciumNitsDir: "/tmp/linting_output/clang-tidy", diff --git a/compiler_wrapper/cros_llvm_next_flags.go b/compiler_wrapper/cros_llvm_next_flags.go index 2211ae18..870e2885 100644 --- a/compiler_wrapper/cros_llvm_next_flags.go +++ b/compiler_wrapper/cros_llvm_next_flags.go @@ -16,6 +16,4 @@ package main // TODO: Enable test in config_test.go, once we have new llvm-next flags. var llvmNextFlags = []string{} -var llvmNextPostFlags = []string{ - "-Wno-unused-but-set-variable", -} +var llvmNextPostFlags = []string{} diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index c3bac4d2..995ce860 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -54,6 +54,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -125,6 +126,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -199,6 +201,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json index 6efe1d2a..86809109 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json @@ -46,6 +46,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -107,6 +108,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -168,6 +170,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -229,6 +232,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -289,6 +293,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -349,6 +354,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -409,6 +415,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -469,6 +476,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -529,6 +537,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json index ac1109c1..2aae9cbb 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json @@ -44,6 +44,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -103,6 +104,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -162,6 +164,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -221,6 +224,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -279,6 +283,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -337,6 +342,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -395,6 +401,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -453,6 +460,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -511,6 +519,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json index d3dc7287..3e396ab9 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json @@ -44,6 +44,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -106,6 +107,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -168,6 +170,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -230,6 +233,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -301,6 +305,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -375,6 +380,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -444,6 +450,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -506,6 +513,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -565,6 +573,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -624,6 +633,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -683,6 +693,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -745,6 +756,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json index 8ef57484..38a34c02 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json @@ -46,6 +46,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -107,6 +108,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -168,6 +170,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -229,6 +232,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -290,6 +294,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -353,6 +358,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -413,6 +419,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -474,6 +481,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json index f93c958b..1098b059 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json @@ -62,6 +62,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -123,6 +124,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -184,6 +186,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -245,6 +248,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json index 5cfe476e..6ea77a71 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json @@ -81,6 +81,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -143,6 +144,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -196,6 +198,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -255,6 +258,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -314,6 +318,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7a-cros-linux-gnueabihf" @@ -373,6 +378,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json index 658ecc26..0d4ee88f 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json @@ -59,6 +59,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -99,6 +100,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -169,6 +171,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -210,6 +213,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -282,6 +286,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -326,6 +331,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -399,6 +405,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -440,6 +447,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json index 14441d7b..fc55b4cd 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json @@ -47,6 +47,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -109,6 +110,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -157,6 +159,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -223,6 +226,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -271,6 +275,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json index 3ffd9e96..ede8dd00 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json @@ -44,6 +44,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -129,6 +130,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -209,6 +211,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -271,6 +274,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index 58a0846f..049722e1 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -53,6 +53,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -120,6 +121,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -190,6 +192,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json index 5865a12f..536797ac 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json @@ -43,6 +43,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -99,6 +100,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -155,6 +157,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -211,6 +214,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -276,6 +280,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -345,6 +350,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -413,6 +419,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -469,6 +476,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -522,6 +530,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -575,6 +584,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -628,6 +638,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -684,6 +695,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json index 658ecc26..0d4ee88f 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json @@ -59,6 +59,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -99,6 +100,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -169,6 +171,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -210,6 +213,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -282,6 +286,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -326,6 +331,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -399,6 +405,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -440,6 +447,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json index aa3de738..c49ac467 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json @@ -46,6 +46,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -102,6 +103,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -144,6 +146,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -204,6 +207,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -246,6 +250,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json index a532745f..f8bed840 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json @@ -44,6 +44,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -124,6 +125,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -204,6 +206,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -266,6 +269,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index 7ccc0f66..cc0d88c9 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -45,6 +45,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -107,6 +108,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -172,6 +174,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json index 36e8f7d0..2a1c799e 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json @@ -37,6 +37,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -89,6 +90,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -141,6 +143,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -194,6 +197,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -245,6 +249,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -297,6 +302,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -349,6 +355,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -400,6 +407,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -452,6 +460,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json index af33ad3a..182e692e 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json @@ -35,6 +35,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -85,6 +86,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -135,6 +137,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -186,6 +189,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -235,6 +239,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -285,6 +290,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -335,6 +341,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -384,6 +391,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -434,6 +442,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json index bb03aa25..c5ff7dae 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json @@ -35,6 +35,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -88,6 +89,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -141,6 +143,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -194,6 +197,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -256,6 +260,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -321,6 +326,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -381,6 +387,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -434,6 +441,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -484,6 +492,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -534,6 +543,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -584,6 +594,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -637,6 +648,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json index 26addbcb..02b873f3 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json @@ -38,6 +38,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -91,6 +92,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -144,6 +146,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -197,6 +200,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -250,6 +254,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -305,6 +310,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -357,6 +363,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -409,6 +416,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json index 2b73d51b..bb1142c9 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json @@ -53,6 +53,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -105,6 +106,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -157,6 +159,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -209,6 +212,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json index 96d6e65a..4d69954b 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json @@ -67,6 +67,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -120,6 +121,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -166,6 +168,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -219,6 +222,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -273,6 +277,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7a-cros-linux-gnueabihf" @@ -323,6 +328,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json index ad6cb480..469678ab 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json @@ -50,6 +50,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -81,6 +82,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -142,6 +144,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -174,6 +177,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -237,6 +241,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -272,6 +277,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -336,6 +342,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -368,6 +375,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json index ba7f61f4..665fddbb 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json @@ -38,6 +38,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -91,6 +92,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -130,6 +132,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -187,6 +190,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -226,6 +230,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json index 5e62ca2e..af220315 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json @@ -35,6 +35,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -106,6 +107,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -172,6 +174,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -225,6 +228,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", -- cgit v1.2.3 From 5200728bf0b8796194d079124a86dcde2f8fedf8 Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Wed, 27 Oct 2021 20:41:33 +0000 Subject: llvm_tools: Add lldb_server to updated_packages This CL adds dev-util/lldb-server to the list of packages updated by various tools. Additionally, it updates the default packages in update_chromeos_llvm_hash.py so that we don't have to manually supply them anymore, and refactors other scripts to use that list instead of maintaining their own. BUG=b:196602751 TEST=`python3 -m unittest discover -s . -p '*_unittest.py'` Change-Id: I0856c8f854dc2cbd0f2f21b71db96d3ef000caa8 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3248929 Tested-by: Ryan Beltran Auto-Submit: Ryan Beltran Commit-Queue: Manoj Gupta Reviewed-by: Manoj Gupta --- llvm_tools/README.md | 1 + llvm_tools/llvm_bisection.py | 9 +++------ llvm_tools/modify_a_tryjob.py | 9 +++------ llvm_tools/update_chromeos_llvm_hash.py | 10 +++++++++- llvm_tools/update_packages_and_run_tests.py | 12 +++--------- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/llvm_tools/README.md b/llvm_tools/README.md index 43c80ad6..74fad6c9 100644 --- a/llvm_tools/README.md +++ b/llvm_tools/README.md @@ -119,6 +119,7 @@ For example, to create a roll CL to the git hash of revision 367622: $ ./update_chromeos_llvm_hash.py \ --update_packages sys-devel/llvm sys-libs/compiler-rt \ sys-libs/libcxx sys-libs/libcxxabi sys-libs/llvm-libunwind \ + 'dev-util/lldb-server' \ --llvm_version 367622 \ --failure_mode disable_patches ``` diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index b1898ea9..baaec703 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -20,9 +20,9 @@ import chroot import get_llvm_hash import git_llvm_rev import modify_a_tryjob +import update_chromeos_llvm_hash import update_tryjob_status - class BisectionExitStatus(enum.Enum): """Exit code when performing bisection.""" @@ -295,10 +295,6 @@ def main(args_output): """ chroot.VerifyOutsideChroot() - update_packages = [ - 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', - 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' - ] patch_metadata_file = 'PATCHES.json' start = args_output.start_rev end = args_output.end_rev @@ -378,7 +374,8 @@ def main(args_output): raise ValueError(f'Revision {rev} exists already in "jobs"') Bisect(revisions, git_hashes, bisect_state, args_output.last_tested, - update_packages, args_output.chroot_path, patch_metadata_file, + update_chromeos_llvm_hash.DEFAULT_PACKAGES, + args_output.chroot_path, patch_metadata_file, args_output.extra_change_lists, args_output.options, args_output.builder, args_output.verbose) diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py index 4d41e6b2..1bca30d4 100755 --- a/llvm_tools/modify_a_tryjob.py +++ b/llvm_tools/modify_a_tryjob.py @@ -17,9 +17,9 @@ import sys import chroot import failure_modes import get_llvm_hash +import update_chromeos_llvm_hash import update_packages_and_run_tests import update_tryjob_status -import update_chromeos_llvm_hash class ModifyTryjob(enum.Enum): @@ -253,17 +253,14 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, # Make sure the revision is within the bounds of the start and end of the # bisection. elif bisect_contents['start'] < revision < bisect_contents['end']: - update_packages = [ - 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', - 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' - ] patch_metadata_file = 'PATCHES.json' git_hash, revision = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( revision) - tryjob_dict = AddTryjob(update_packages, git_hash, revision, chroot_path, + tryjob_dict = AddTryjob(update_chromeos_llvm_hash.DEFAULT_PACKAGES, + git_hash, revision, chroot_path, patch_metadata_file, extra_cls, options, builder, verbose, revision) diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py index ab83321a..9365f00b 100755 --- a/llvm_tools/update_chromeos_llvm_hash.py +++ b/llvm_tools/update_chromeos_llvm_hash.py @@ -25,6 +25,14 @@ import get_llvm_hash import git import llvm_patch_management +DEFAULT_PACKAGES = [ + 'dev-util/lldb-server', + 'sys-devel/llvm', + 'sys-libs/compiler-rt', + 'sys-libs/libcxx', + 'sys-libs/libcxxabi', + 'sys-libs/llvm-libunwind', +] # Specify which LLVM hash to update class LLVMVariant(enum.Enum): @@ -77,7 +85,7 @@ def GetCommandLineArgs(): # Add argument for specific builds to uprev and update their llvm-next hash. parser.add_argument( '--update_packages', - default=['sys-devel/llvm'], + default=DEFAULT_PACKAGES, required=False, nargs='+', help='the ebuilds to update their hash for llvm-next ' diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py index dd01253e..92f49307 100755 --- a/llvm_tools/update_packages_and_run_tests.py +++ b/llvm_tools/update_packages_and_run_tests.py @@ -21,7 +21,6 @@ import update_chromeos_llvm_hash VALID_CQ_TRYBOTS = ['llvm', 'llvm-next', 'llvm-tot'] - def GetCommandLineArgs(): """Parses the command line for the command line arguments. @@ -402,11 +401,6 @@ def main(): args_output = GetCommandLineArgs() - update_packages = [ - 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', - 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' - ] - patch_metadata_file = 'PATCHES.json' svn_option = args_output.llvm_version @@ -420,8 +414,8 @@ def main(): # If --last_tested is specified, check if the current run has the same # arguments last time --last_tested is used. if args_output.last_tested: - chroot_file_paths = chroot.GetChrootEbuildPaths(args_output.chroot_path, - update_packages) + chroot_file_paths = chroot.GetChrootEbuildPaths( + args_output.chroot_path, update_chromeos_llvm_hash.DEFAULT_PACKAGES) arg_dict = { 'svn_version': svn_version, 'ebuilds': chroot_file_paths, @@ -449,7 +443,7 @@ def main(): extra_commit_msg += cq_trybot_msg change_list = update_chromeos_llvm_hash.UpdatePackages( - update_packages, + update_chromeos_llvm_hash.DEFAULT_PACKAGES, llvm_variant, git_hash, svn_version, -- cgit v1.2.3 From d080198a6e20a85e3d6c57a9fb168d4e19b3503c Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Wed, 27 Oct 2021 20:47:16 +0000 Subject: llvm_tools: fix yapf warnings This CL applies fixes automatically generated by yapf during presubmit hook as well as a few manual fixes: * Removing erroneous `verbose` argument from call to RunTryJobs * Replacing / with parenthsis for long lines * Shortened some variable names to fix lines that were too long BUG=None TEST=rerean unit tests Change-Id: Ic6ed4bb74f067dd5c67991481caf75d8829bf86e Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3248930 Tested-by: Ryan Beltran Auto-Submit: Ryan Beltran Commit-Queue: Manoj Gupta Reviewed-by: Manoj Gupta --- llvm_tools/llvm_bisection.py | 148 +++++++++++++--------------- llvm_tools/modify_a_tryjob.py | 66 ++++++------- llvm_tools/update_chromeos_llvm_hash.py | 67 ++++++------- llvm_tools/update_packages_and_run_tests.py | 90 ++++++++--------- 4 files changed, 179 insertions(+), 192 deletions(-) diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index baaec703..0148efd2 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -23,6 +23,7 @@ import modify_a_tryjob import update_chromeos_llvm_hash import update_tryjob_status + class BisectionExitStatus(enum.Enum): """Exit code when performing bisection.""" @@ -51,18 +52,16 @@ def GetCommandLineArgs(): 'the first bad version (default: %(default)s)') # Add argument for the good LLVM revision for bisection. - parser.add_argument( - '--start_rev', - required=True, - type=int, - help='The good revision for the bisection.') + parser.add_argument('--start_rev', + required=True, + type=int, + help='The good revision for the bisection.') # Add argument for the bad LLVM revision for bisection. - parser.add_argument( - '--end_rev', - required=True, - type=int, - help='The bad revision for the bisection.') + parser.add_argument('--end_rev', + required=True, + type=int, + help='The bad revision for the bisection.') # Add argument for the absolute path to the file that contains information on # the previous tested svn version. @@ -88,42 +87,38 @@ def GetCommandLineArgs(): 'of updating the packages') # Add argument for custom options for the tryjob. - parser.add_argument( - '--options', - required=False, - nargs='+', - help='options to use for the tryjob testing') + parser.add_argument('--options', + required=False, + nargs='+', + help='options to use for the tryjob testing') # Add argument for the builder to use for the tryjob. - parser.add_argument( - '--builder', required=True, help='builder to use for the tryjob testing') + parser.add_argument('--builder', + required=True, + help='builder to use for the tryjob testing') # Add argument for the description of the tryjob. - parser.add_argument( - '--description', - required=False, - nargs='+', - help='the description of the tryjob') + parser.add_argument('--description', + required=False, + nargs='+', + help='the description of the tryjob') # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=cros_root, - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--nocleanup', - action='store_false', - dest='cleanup', - help='Abandon CLs created for bisectoin') + parser.add_argument('--nocleanup', + action='store_false', + dest='cleanup', + help='Abandon CLs created for bisectoin') args_output = parser.parse_args() @@ -174,8 +169,7 @@ def GetRemainingRange(start, end, tryjobs): all_bad_revisions = [end] all_bad_revisions.extend( - cur_tryjob['rev'] - for cur_tryjob in tryjobs + cur_tryjob['rev'] for cur_tryjob in tryjobs if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.BAD.value) # The minimum value for the 'bad' field in the tryjobs is the new end @@ -184,8 +178,7 @@ def GetRemainingRange(start, end, tryjobs): all_good_revisions = [start] all_good_revisions.extend( - cur_tryjob['rev'] - for cur_tryjob in tryjobs + cur_tryjob['rev'] for cur_tryjob in tryjobs if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.GOOD.value) # The maximum value for the 'good' field in the tryjobs is the new start @@ -205,8 +198,8 @@ def GetRemainingRange(start, end, tryjobs): pending_revisions = { tryjob['rev'] for tryjob in tryjobs - if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value and - good_rev < tryjob['rev'] < bad_rev + if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value + and good_rev < tryjob['rev'] < bad_rev } # Find all revisions that are to be skipped within 'good_rev' and 'bad_rev'. @@ -217,8 +210,8 @@ def GetRemainingRange(start, end, tryjobs): skip_revisions = { tryjob['rev'] for tryjob in tryjobs - if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value and - good_rev < tryjob['rev'] < bad_rev + if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value + and good_rev < tryjob['rev'] < bad_rev } return good_rev, bad_rev, pending_revisions, skip_revisions @@ -301,56 +294,56 @@ def main(args_output): bisect_state = LoadStatusFile(args_output.last_tested, start, end) if start != bisect_state['start'] or end != bisect_state['end']: - raise ValueError(f'The start {start} or the end {end} version provided is ' - f'different than "start" {bisect_state["start"]} or "end" ' - f'{bisect_state["end"]} in the .JSON file') + raise ValueError( + f'The start {start} or the end {end} version provided is ' + f'different than "start" {bisect_state["start"]} or "end" ' + f'{bisect_state["end"]} in the .JSON file') - # Pending and skipped revisions are between 'start_revision' and - # 'end_revision'. - start_revision, end_revision, pending_revisions, skip_revisions = \ - GetRemainingRange(start, end, bisect_state['jobs']) + # Pending and skipped revisions are between 'start_rev' and 'end_rev'. + start_rev, end_rev, pending_revs, skip_revs = GetRemainingRange( + start, end, bisect_state['jobs']) - revisions, git_hashes = GetCommitsBetween(start_revision, end_revision, + revisions, git_hashes = GetCommitsBetween(start_rev, end_rev, args_output.parallel, - args_output.src_path, - pending_revisions, skip_revisions) + args_output.src_path, pending_revs, + skip_revs) - # No more revisions between 'start_revision' and 'end_revision', so + # No more revisions between 'start_rev' and 'end_rev', so # bisection is complete. # - # This is determined by finding all valid revisions between 'start_revision' - # and 'end_revision' and that are NOT in the 'pending' and 'skipped' set. + # This is determined by finding all valid revisions between 'start_rev' + # and 'end_rev' and that are NOT in the 'pending' and 'skipped' set. if not revisions: - if pending_revisions: + if pending_revs: # Some tryjobs are not finished which may change the actual bad # commit/revision when those tryjobs are finished. - no_revisions_message = (f'No revisions between start {start_revision} ' - f'and end {end_revision} to create tryjobs\n') + no_revisions_message = (f'No revisions between start {start_rev} ' + f'and end {end_rev} to create tryjobs\n') - if pending_revisions: - no_revisions_message += ( - 'The following tryjobs are pending:\n' + - '\n'.join(str(rev) for rev in pending_revisions) + '\n') + if pending_revs: + no_revisions_message += ('The following tryjobs are pending:\n' + + '\n'.join(str(rev) + for rev in pending_revs) + '\n') - if skip_revisions: + if skip_revs: no_revisions_message += ('The following tryjobs were skipped:\n' + - '\n'.join(str(rev) for rev in skip_revisions) + - '\n') + '\n'.join(str(rev) + for rev in skip_revs) + '\n') raise ValueError(no_revisions_message) print(f'Finished bisecting for {args_output.last_tested}') if args_output.src_path: bad_llvm_hash = get_llvm_hash.GetGitHashFrom(args_output.src_path, - end_revision) + end_rev) else: - bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_revision) - print(f'The bad revision is {end_revision} and its commit hash is ' + bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_rev) + print(f'The bad revision is {end_rev} and its commit hash is ' f'{bad_llvm_hash}') - if skip_revisions: - skip_revisions_message = ('\nThe following revisions were skipped:\n' + - '\n'.join(str(rev) for rev in skip_revisions)) - print(skip_revisions_message) + if skip_revs: + skip_revs_message = ('\nThe following revisions were skipped:\n' + + '\n'.join(str(rev) for rev in skip_revs)) + print(skip_revs_message) if args_output.cleanup: # Abandon all the CLs created for bisection @@ -374,10 +367,9 @@ def main(args_output): raise ValueError(f'Revision {rev} exists already in "jobs"') Bisect(revisions, git_hashes, bisect_state, args_output.last_tested, - update_chromeos_llvm_hash.DEFAULT_PACKAGES, - args_output.chroot_path, patch_metadata_file, - args_output.extra_change_lists, args_output.options, - args_output.builder, args_output.verbose) + update_chromeos_llvm_hash.DEFAULT_PACKAGES, args_output.chroot_path, + patch_metadata_file, args_output.extra_change_lists, + args_output.options, args_output.builder, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py index 1bca30d4..519fb51e 100755 --- a/llvm_tools/modify_a_tryjob.py +++ b/llvm_tools/modify_a_tryjob.py @@ -57,11 +57,10 @@ def GetCommandLineArgs(): # Add argument that determines which revision to search for in the list of # tryjobs. - parser.add_argument( - '--revision', - required=True, - type=int, - help='The revision to either remove or relaunch.') + parser.add_argument('--revision', + required=True, + type=int, + help='The revision to either remove or relaunch.') # Add argument for other change lists that want to run alongside the tryjob. parser.add_argument( @@ -72,40 +71,38 @@ def GetCommandLineArgs(): 'of updating the packages') # Add argument for custom options for the tryjob. - parser.add_argument( - '--options', - required=False, - nargs='+', - help='options to use for the tryjob testing') + parser.add_argument('--options', + required=False, + nargs='+', + help='options to use for the tryjob testing') # Add argument for the builder to use for the tryjob. - parser.add_argument('--builder', help='builder to use for the tryjob testing') + parser.add_argument('--builder', + help='builder to use for the tryjob testing') # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=cros_root, - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') args_output = parser.parse_args() - if not os.path.isfile(args_output.status_file) or \ - not args_output.status_file.endswith('.json'): + if (not os.path.isfile(args_output.status_file) + or not args_output.status_file.endswith('.json')): raise ValueError('File does not exist or does not ending in ".json" ' ': %s' % args_output.status_file) - if args_output.modify_tryjob == ModifyTryjob.ADD.value and \ - not args_output.builder: + if (args_output.modify_tryjob == ModifyTryjob.ADD.value + and not args_output.builder): raise ValueError('A builder is required for adding a tryjob.') - elif args_output.modify_tryjob != ModifyTryjob.ADD.value and \ - args_output.builder: + elif (args_output.modify_tryjob != ModifyTryjob.ADD.value + and args_output.builder): raise ValueError('Specifying a builder is only available when adding a ' 'tryjob.') @@ -234,13 +231,13 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, bisect_contents['jobs'][tryjob_index]['cl'], bisect_contents['jobs'][tryjob_index]['extra_cls'], bisect_contents['jobs'][tryjob_index]['options'], - bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, verbose) + bisect_contents['jobs'][tryjob_index]['builder'], chroot_path) bisect_contents['jobs'][tryjob_index][ 'status'] = update_tryjob_status.TryjobStatus.PENDING.value bisect_contents['jobs'][tryjob_index]['link'] = tryjob_results[0]['link'] - bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[0][ - 'buildbucket_id'] + bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[ + 0]['buildbucket_id'] print('Successfully relaunched the tryjob for revision %d and updated ' 'the tryjob link to %s' % (revision, tryjob_results[0]['link'])) @@ -274,7 +271,10 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, modify_tryjob) with open(status_file, 'w') as update_tryjobs: - json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': ')) + json.dump(bisect_contents, + update_tryjobs, + indent=4, + separators=(',', ': ')) def main(): @@ -287,9 +287,9 @@ def main(): PerformTryjobModification(args_output.revision, ModifyTryjob(args_output.modify_tryjob), args_output.status_file, - args_output.extra_change_lists, args_output.options, - args_output.builder, args_output.chroot_path, - args_output.verbose) + args_output.extra_change_lists, + args_output.options, args_output.builder, + args_output.chroot_path, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py index 9365f00b..4e9b9104 100755 --- a/llvm_tools/update_chromeos_llvm_hash.py +++ b/llvm_tools/update_chromeos_llvm_hash.py @@ -34,6 +34,7 @@ DEFAULT_PACKAGES = [ 'sys-libs/llvm-libunwind', ] + # Specify which LLVM hash to update class LLVMVariant(enum.Enum): """Represent the LLVM hash in an ebuild file to update.""" @@ -77,26 +78,23 @@ def GetCommandLineArgs(): description="Updates the build's hash for llvm-next.") # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=defaultCrosRoot(), - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=defaultCrosRoot(), + help='the path to the chroot (default: %(default)s)') # Add argument for specific builds to uprev and update their llvm-next hash. - parser.add_argument( - '--update_packages', - default=DEFAULT_PACKAGES, - required=False, - nargs='+', - help='the ebuilds to update their hash for llvm-next ' - '(default: %(default)s)') + parser.add_argument('--update_packages', + default=DEFAULT_PACKAGES, + required=False, + nargs='+', + help='the ebuilds to update their hash for llvm-next ' + '(default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for the LLVM hash to update parser.add_argument( @@ -322,9 +320,11 @@ def UprevEbuildToVersion(symlink, svn_version, git_hash): count=1) # any other package else: - new_ebuild, is_changed = re.subn( - r'(\d+)\.(\d+)_pre([0-9]+)', - '%s.\\2_pre%s' % (llvm_major_version, svn_version), ebuild, count=1) + new_ebuild, is_changed = re.subn(r'(\d+)\.(\d+)_pre([0-9]+)', + '%s.\\2_pre%s' % + (llvm_major_version, svn_version), + ebuild, + count=1) if not is_changed: # failed to increment the revision number raise ValueError('Failed to uprev the ebuild.') @@ -405,7 +405,8 @@ def StagePatchMetadataFileForCommit(patch_metadata_file_path): # Cmd to stage the patch metadata file for commit. subprocess.check_output([ 'git', '-C', - os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path + os.path.dirname(patch_metadata_file_path), 'add', + patch_metadata_file_path ]) @@ -427,9 +428,9 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): # changed, if so, add which patches have changed to the commit # message. for package_name, patch_info_dict in package_info_dict.items(): - if (patch_info_dict['disabled_patches'] or - patch_info_dict['removed_patches'] or - patch_info_dict['modified_metadata']): + if (patch_info_dict['disabled_patches'] + or patch_info_dict['removed_patches'] + or patch_info_dict['modified_metadata']): cur_package_header = '\nFor the package %s:' % package_name commit_messages.append(cur_package_header) @@ -608,16 +609,16 @@ def main(): git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( git_hash_source) - change_list = UpdatePackages( - args_output.update_packages, - llvm_variant, - git_hash, - svn_version, - args_output.chroot_path, - args_output.patch_metadata_file, - failure_modes.FailureModes(args_output.failure_mode), - git_hash_source, - extra_commit_msg=None) + change_list = UpdatePackages(args_output.update_packages, + llvm_variant, + git_hash, + svn_version, + args_output.chroot_path, + args_output.patch_metadata_file, + failure_modes.FailureModes( + args_output.failure_mode), + git_hash_source, + extra_commit_msg=None) print('Successfully updated packages to %s (%d)' % (git_hash, svn_version)) print('Gerrit URL: %s' % change_list.url) diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py index 92f49307..2e4a9058 100755 --- a/llvm_tools/update_packages_and_run_tests.py +++ b/llvm_tools/update_packages_and_run_tests.py @@ -21,6 +21,7 @@ import update_chromeos_llvm_hash VALID_CQ_TRYBOTS = ['llvm', 'llvm-next', 'llvm-tot'] + def GetCommandLineArgs(): """Parses the command line for the command line arguments. @@ -50,10 +51,9 @@ def GetCommandLineArgs(): 'of updating the packages') # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=cros_root, - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') # Add argument to choose between llvm and llvm-next. parser.add_argument( @@ -70,65 +70,58 @@ def GetCommandLineArgs(): 'arguments.') # Add argument for the LLVM version to use. - parser.add_argument( - '--llvm_version', - type=get_llvm_hash.IsSvnOption, - required=True, - help='which git hash of LLVM to find ' - '{google3, ToT, } ' - '(default: finds the git hash of the google3 LLVM ' - 'version)') + parser.add_argument('--llvm_version', + type=get_llvm_hash.IsSvnOption, + required=True, + help='which git hash of LLVM to find ' + '{google3, ToT, } ' + '(default: finds the git hash of the google3 LLVM ' + 'version)') # Add argument to add reviewers for the created CL. - parser.add_argument( - '--reviewers', - nargs='+', - default=[], - help='The reviewers for the package update changelist') + parser.add_argument('--reviewers', + nargs='+', + default=[], + help='The reviewers for the package update changelist') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') subparsers = parser.add_subparsers(dest='subparser_name') subparser_names = [] # Testing with the tryjobs. tryjob_subparser = subparsers.add_parser('tryjobs') subparser_names.append('tryjobs') - tryjob_subparser.add_argument( - '--builders', - required=True, - nargs='+', - default=[], - help='builders to use for the tryjob testing') + tryjob_subparser.add_argument('--builders', + required=True, + nargs='+', + default=[], + help='builders to use for the tryjob testing') # Add argument for custom options for the tryjob. - tryjob_subparser.add_argument( - '--options', - required=False, - nargs='+', - default=[], - help='options to use for the tryjob testing') + tryjob_subparser.add_argument('--options', + required=False, + nargs='+', + default=[], + help='options to use for the tryjob testing') # Testing with the recipe builders recipe_subparser = subparsers.add_parser('recipe') subparser_names.append('recipe') - recipe_subparser.add_argument( - '--options', - required=False, - nargs='+', - default=[], - help='options passed to the recipe builders') - - recipe_subparser.add_argument( - '--builders', - required=True, - nargs='+', - default=[], - help='recipe builders to launch') + recipe_subparser.add_argument('--options', + required=False, + nargs='+', + default=[], + help='options passed to the recipe builders') + + recipe_subparser.add_argument('--builders', + required=True, + nargs='+', + default=[], + help='recipe builders to launch') # Testing with CQ. cq_subparser = subparsers.add_parser('cq') @@ -359,7 +352,8 @@ def GetCQDependString(dependent_cls): return None # Cq-Depend must start a new paragraph prefixed with "Cq-Depend". - return '\nCq-Depend: ' + ', '.join(('chromium:%s' % i) for i in dependent_cls) + return '\nCq-Depend: ' + ', '.join( + ('chromium:%s' % i) for i in dependent_cls) def GetCQIncludeTrybotsString(trybot): -- cgit v1.2.3 From 70c83ac3df5d50f4fc4612fa2c9a5b8af4455842 Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Tue, 26 Oct 2021 20:26:26 +0000 Subject: Replace non-inclusive language Wherever possible, this commit removes non-inclusive languages as per COIL policy. Some of this language exists in dead code, so this commit also removes confirmed dead code. Also some minor lint fixes to get the code past the lint checks. BUG=b:204057159 TEST=Presubmit tests Change-Id: I299bfbe5dfcbe761efc5dcb29caea2337abe3f30 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3243940 Tested-by: Jordan R Abrahams Reviewed-by: Manoj Gupta Reviewed-by: George Burgess --- android_merge_from_upstream.sh | 13 +- compiler_wrapper/sanitizer_flags.go | 2 +- cros_utils/manifest_versions.py | 162 ------------- cros_utils/misc.py | 109 +++------ crosperf/results_cache.py | 99 ++++---- crosperf/results_cache_unittest.py | 190 +++++++++------- crosperf/test_cache/compare_output/results.pickle | Bin 0 -> 8081 bytes crosperf/test_cache/compare_output/results.txt | Bin 8124 -> 0 bytes crosperf/test_cache/test_input/results.pickle | 6 + crosperf/test_cache/test_input/results.txt | 6 - .../test_puretelemetry_input/results.pickle | 6 + .../test_puretelemetry_input/results.txt | 6 - llvm_extra/create_llvm_extra.sh | 2 +- llvm_tools/nightly_revert_checker.py | 97 ++++---- rust_tools/rust_watch.py | 21 +- setup_chromeos.py | 252 --------------------- unblocked_terms.txt | 5 - 17 files changed, 270 insertions(+), 706 deletions(-) delete mode 100644 cros_utils/manifest_versions.py create mode 100644 crosperf/test_cache/compare_output/results.pickle delete mode 100644 crosperf/test_cache/compare_output/results.txt create mode 100644 crosperf/test_cache/test_input/results.pickle delete mode 100644 crosperf/test_cache/test_input/results.txt create mode 100644 crosperf/test_cache/test_puretelemetry_input/results.pickle delete mode 100644 crosperf/test_cache/test_puretelemetry_input/results.txt delete mode 100755 setup_chromeos.py delete mode 100644 unblocked_terms.txt diff --git a/android_merge_from_upstream.sh b/android_merge_from_upstream.sh index b63eb6e1..7430b8dd 100755 --- a/android_merge_from_upstream.sh +++ b/android_merge_from_upstream.sh @@ -11,9 +11,9 @@ # https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/1132504/1 local_branch_name="merge_with_upstream" -local_upstream="aosp/master" +local_upstream="aosp/master" # nocheck remote="aosp" -remote_branch="${remote}/upstream-mirror-master" +remote_branch="${remote}/upstream-mirror-master" # nocheck my_dir="$(dirname "$(readlink -m "$0")")" cd "${my_dir}" @@ -21,7 +21,7 @@ cd "${my_dir}" ensure_head_is_upstream_main() { local current_rev main_rev current_rev="$(git rev-parse HEAD)" - main_rev="$(git rev-parse ${local_upstream})" + main_rev="$(git rev-parse "${local_upstream}")" if [[ "${current_rev}" != "${main_rev}" ]]; then echo "Please checkout ${local_upstream} and rerun this" >&2 exit @@ -49,7 +49,7 @@ ensure_no_local_branch_present() { get_merge_commit_list() { local merge_base - merge_base="$(git merge-base HEAD ${remote_branch})" + merge_base="$(git merge-base HEAD "${remote_branch}")" git log --oneline "${merge_base}..${remote_branch}" } @@ -62,6 +62,9 @@ repo start "${local_branch_name}" commit_list="$(get_merge_commit_list)" num_commits="$(wc -l <<< "${commit_list}")" + +# Disable shellcheck for the sed substitution warning. +# shellcheck disable=SC2001 commit_message="Merging ${num_commits} commit(s) from Chromium's toolchain-utils Merged commit digest: @@ -69,6 +72,6 @@ $(sed 's/^/ /' <<< "${commit_list}") " git merge "${remote_branch}" -m "${commit_message}" -echo 'NOTE: When you try to `repo upload`, repo might show a scary warning' +echo 'NOTE: When you try to "repo upload", repo might show a scary warning' echo 'about the number of changes are being uploaded. That should be fine,' echo 'since repo will only create CLs for commits not known to our remote.' diff --git a/compiler_wrapper/sanitizer_flags.go b/compiler_wrapper/sanitizer_flags.go index fe8d1503..abb385a8 100644 --- a/compiler_wrapper/sanitizer_flags.go +++ b/compiler_wrapper/sanitizer_flags.go @@ -14,7 +14,7 @@ func processSanitizerFlags(builder *commandBuilder) { hasSanitizeFuzzerFlags := false for _, arg := range builder.args { // TODO: This should probably be -fsanitize= to not match on - // e.g. -fsanitize-blacklist + // e.g. -fsanitize-blocklist if arg.fromUser { if strings.HasPrefix(arg.value, "-fsanitize") { hasSanitizeFlags = true diff --git a/cros_utils/manifest_versions.py b/cros_utils/manifest_versions.py deleted file mode 100644 index 4838de3c..00000000 --- a/cros_utils/manifest_versions.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tools for searching/manipulating the manifests repository.""" - -from __future__ import print_function - -__author__ = 'llozano@google.com (Luis Lozano)' - -import copy -import os -import re -import shutil -import tempfile -import time - -from cros_utils import command_executer -from cros_utils import logger - -MANIFEST_VERSION_MAIN_BRANCH = 'master' - - -def IsCrosVersion(version): - match = re.search(r'(\d+\.\d+\.\d+\.\d+)', version) - return match is not None - - -def IsRFormatCrosVersion(version): - match = re.search(r'(R\d+-\d+\.\d+\.\d+)', version) - return match is not None - - -def RFormatCrosVersion(version): - assert IsCrosVersion(version) - tmp_major, tmp_minor = version.split('.', 1) - rformat = 'R' + tmp_major + '-' + tmp_minor - assert IsRFormatCrosVersion(rformat) - return rformat - - -class ManifestVersions(object): - """This class handles interactions with the manifests repo.""" - - def __init__(self, internal=True): - self.internal = internal - self.clone_location = tempfile.mkdtemp() - self.ce = command_executer.GetCommandExecuter() - if internal: - versions_git = ('https://chrome-internal.googlesource.com/' - 'chromeos/manifest-versions.git') - else: - versions_git = ( - 'https://chromium.googlesource.com/chromiumos/manifest-versions.git') - commands = [ - 'cd {0}'.format(self.clone_location), - 'git clone {0}'.format(versions_git) - ] - ret = self.ce.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal('Failed to clone manifest-versions.') - - def __del__(self): - if self.clone_location: - shutil.rmtree(self.clone_location) - - def TimeToVersionChromeOS(self, my_time): - """Convert timestamp to version number, in ChromeOS/Paladin.""" - cur_time = time.mktime(time.gmtime()) - des_time = float(my_time) - if cur_time - des_time > 7000000: - logger.GetLogger().LogFatal('The time you specify is too early.') - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout -f $(git rev-list' + - ' --max-count=1 --before={0} origin/{1})'.format( - my_time, MANIFEST_VERSION_MAIN_BRANCH) - ] - ret = self.ce.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal('Failed to checkout manifest at ' - 'specified time') - path = os.path.realpath('{0}/manifest-versions/LKGM/lkgm.xml'.format( - self.clone_location)) - pp = path.split('/') - new_list = copy.deepcopy(pp) - for i, e in enumerate(pp): - if e == 'android-LKGM-candidates': - new_list[i] = 'paladin' - chrome_path = '/'.join(new_list) - if not os.path.exists(chrome_path): - logger.GetLogger().LogOutput('LKGM path is %s' % path) - logger.GetLogger().LogOutput('Cannot find path %s' % chrome_path) - pieces = os.path.basename(chrome_path).split('.') - pieces = pieces[:-2] - new_base = '.'.join(pieces) + '*' - wild_path = os.path.join('/', '/'.join(new_list[:-1]), new_base) - command = 'ls %s' % wild_path - ret, out, _ = self.ce.RunCommandWOutput(command) - if ret == 0: - out = out.strip() - files = out.split('\n') - latest = files[-1] - small = os.path.basename(latest).split('.xml')[0] - version = pp[-2] + '.' + small - else: - small = os.path.basename(path).split('.xml')[0] - version = pp[-2] + '.' + small - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH) - ] - self.ce.RunCommands(commands) - return version - - def TimeToVersion(self, my_time): - """Convert timestamp to version number.""" - cur_time = time.mktime(time.gmtime()) - des_time = float(my_time) - if cur_time - des_time > 7000000: - logger.GetLogger().LogFatal('The time you specify is too early.') - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout -f $(git rev-list' + - ' --max-count=1 --before={0} origin/{1})'.format( - my_time, MANIFEST_VERSION_MAIN_BRANCH) - ] - ret = self.ce.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal('Failed to checkout manifest at ' - 'specified time') - path = os.path.realpath('{0}/manifest-versions/LKGM/lkgm.xml'.format( - self.clone_location)) - pp = path.split('/') - small = os.path.basename(path).split('.xml')[0] - version = pp[-2] + '.' + small - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH) - ] - self.ce.RunCommands(commands) - return version - - def GetManifest(self, version, to_file): - """Get the manifest file from a given chromeos-internal version.""" - assert not IsRFormatCrosVersion(version) - version = version.split('.', 1)[1] - os.chdir(self.clone_location) - files = [ - os.path.join(r, f) - for r, _, fs in os.walk('.') - for f in fs - if version in f - ] - if files: - command = 'cp {0} {1}'.format(files[0], to_file) - ret = self.ce.RunCommand(command) - if ret: - raise RuntimeError('Cannot copy manifest to {0}'.format(to_file)) - else: - raise RuntimeError('Version {0} is not available.'.format(version)) diff --git a/cros_utils/misc.py b/cros_utils/misc.py index 93d1b3b6..a0d0de73 100644 --- a/cros_utils/misc.py +++ b/cros_utils/misc.py @@ -15,7 +15,6 @@ import os import re import shutil import sys -import traceback from cros_utils import command_executer from cros_utils import logger @@ -24,8 +23,6 @@ CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts' TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/' 'cros_utils/toolchain_utils.sh') -CROS_MAIN_BRANCH = 'cros/master' - def GetChromeOSVersionFromLSBVersion(lsb_version): """Get Chromeos version from Lsb version.""" @@ -112,8 +109,8 @@ def FormatQuotedCommand(command): def FormatCommands(commands): - return ApplySubs( - str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n')) + return ApplySubs(str(commands), ('&&', '&&\n'), (';', ';\n'), + (r'\n+\s*', '\n')) def GetImageDir(chromeos_root, board): @@ -155,16 +152,16 @@ def GetBuildPackagesCommand(board, usepkg=False, debug=False): withdebug_flag = '--nowithdebug' return ('%s/build_packages %s --withdev --withtest --withautotest ' '--skip_toolchain_update %s --board=%s ' - '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag, - withdebug_flag, board)) + '--accept_licenses=@CHROMEOS' % + (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board)) def GetBuildImageCommand(board, dev=False): dev_args = '' if dev: dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs' - return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board, - dev_args)) + return ('%s/build_image --board=%s %s test' % + (CHROMEOS_SCRIPTS_DIR, board, dev_args)) def GetSetupBoardCommand(board, usepkg=None, force=None): @@ -193,8 +190,8 @@ def CanonicalizePath(path): def GetCtargetFromBoard(board, chromeos_root): """Get Ctarget from board.""" base_board = board.split('_')[0] - command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH, - base_board)) + command = ('source %s; get_ctarget_from_board %s' % + (TOOLCHAIN_UTILS_PATH, base_board)) ce = command_executer.GetCommandExecuter() ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) if ret != 0: @@ -207,8 +204,8 @@ def GetCtargetFromBoard(board, chromeos_root): def GetArchFromBoard(board, chromeos_root): """Get Arch from board.""" base_board = board.split('_')[0] - command = ( - 'source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH, base_board)) + command = ('source %s; get_board_arch %s' % + (TOOLCHAIN_UTILS_PATH, base_board)) ce = command_executer.GetCommandExecuter() ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) if ret != 0: @@ -316,23 +313,22 @@ def WorkingDirectory(new_dir): def HasGitStagedChanges(git_dir): """Return True if git repository has staged changes.""" - command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format( - git_dir) + command = f'cd {git_dir} && git diff --quiet --cached --exit-code HEAD' return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False) def HasGitUnstagedChanges(git_dir): """Return True if git repository has un-staged changes.""" - command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir) + command = f'cd {git_dir} && git diff --quiet --exit-code HEAD' return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False) def HasGitUntrackedChanges(git_dir): """Return True if git repository has un-tracked changes.""" - command = ('cd {0} && test -z ' - '$(git ls-files --exclude-standard --others)').format(git_dir) + command = (f'cd {git_dir} && test -z ' + '$(git ls-files --exclude-standard --others)') return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False) @@ -352,8 +348,8 @@ def GitGetCommitHash(git_dir, commit_symbolic_name): The git hash for the symbolic name or None if fails. """ - command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format( - git_dir, commit_symbolic_name) + command = (f'cd {git_dir} && git log -n 1' + f' --pretty="format:%H" {commit_symbolic_name}') rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( command, print_to_console=False) if rv == 0: @@ -393,7 +389,7 @@ def GetGitChangesAsList(git_dir, path=None, staged=False): Returns: A list containing all the changed files. """ - command = 'cd {0} && git diff --name-only'.format(git_dir) + command = f'cd {git_dir} && git diff --name-only' if staged: command += ' --cached' if path: @@ -408,8 +404,8 @@ def GetGitChangesAsList(git_dir, path=None, staged=False): def IsChromeOsTree(chromeos_root): return (os.path.isdir( - os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and - os.path.isdir(os.path.join(chromeos_root, 'manifest'))) + os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) + and os.path.isdir(os.path.join(chromeos_root, 'manifest'))) def DeleteChromeOsTree(chromeos_root, dry_run=False): @@ -423,11 +419,10 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): True if everything is ok. """ if not IsChromeOsTree(chromeos_root): - logger.GetLogger().LogWarning( - '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format( - chromeos_root)) + logger.GetLogger().LogWarning(f'"{chromeos_root}" does not seem to be a' + ' valid chromeos tree, do nothing.') return False - cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root) + cmd0 = f'cd {chromeos_root} && cros_sdk --delete' if dry_run: print(cmd0) else: @@ -435,10 +430,10 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): cmd0, print_to_console=True) != 0: return False - cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && ' - 'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && ' - 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME' - ).format(chromeos_root) + cmd1 = ( + f'export CHROMEOSDIRNAME="$(dirname $(cd {chromeos_root} && pwd))" && ' + f'export CHROMEOSBASENAME="$(basename $(cd {chromeos_root} && pwd))" && ' + 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME') if dry_run: print(cmd1) return True @@ -447,54 +442,6 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): cmd1, print_to_console=True) == 0 -def ApplyGerritPatches(chromeos_root, - gerrit_patch_string, - branch=CROS_MAIN_BRANCH): - """Apply gerrit patches on a chromeos tree. - - Args: - chromeos_root: chromeos tree path - gerrit_patch_string: a patch string just like the one gives to cbuildbot, - 'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch. - branch: the tree based on which to apply the patches. - - Returns: - True if success. - """ - - ### First of all, we need chromite libs - sys.path.append(os.path.join(chromeos_root, 'chromite')) - # Imports below are ok after modifying path to add chromite. - # Pylint cannot detect that and complains. - # pylint: disable=import-error, import-outside-toplevel - from lib import git - from lib import gerrit - manifest = git.ManifestCheckout(chromeos_root) - patch_list = gerrit_patch_string.split(' ') - ### This takes time, print log information. - logger.GetLogger().LogOutput('Retrieving patch information from server ...') - patch_info_list = gerrit.GetGerritPatchInfo(patch_list) - for pi in patch_info_list: - project_checkout = manifest.FindCheckout(pi.project, strict=False) - if not project_checkout: - logger.GetLogger().LogError( - 'Failed to find patch project "{project}" in manifest.'.format( - project=pi.project)) - return False - - pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref) - try: - project_git_path = project_checkout.GetPath(absolute=True) - logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format( - pi_str, project_git_path)) - pi.Apply(project_git_path, branch, trivial=False) - except Exception: - traceback.print_exc(file=sys.stdout) - logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str)) - return False - return True - - def BooleanPrompt(prompt='Do you want to continue?', default=True, true_value='yes', @@ -515,8 +462,8 @@ def BooleanPrompt(prompt='Do you want to continue?', true_value, false_value = true_value.lower(), false_value.lower() true_text, false_text = true_value, false_value if true_value == false_value: - raise ValueError( - 'true_value and false_value must differ: got %r' % true_value) + raise ValueError('true_value and false_value must differ: got %r' % + true_value) if default: true_text = true_text[0].upper() + true_text[1:] diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 87e30ecc..5525858c 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -27,7 +27,7 @@ import results_report import test_flag SCRATCH_DIR = os.path.expanduser('~/cros_scratch') -RESULTS_FILE = 'results.txt' +RESULTS_FILE = 'results.pickle' MACHINE_FILE = 'machine.txt' AUTOTEST_TARBALL = 'autotest.tbz2' RESULTS_TARBALL = 'results.tbz2' @@ -197,9 +197,9 @@ class Result(object): keyvals_dict[key] = result_dict['value'] elif 'values' in result_dict: values = result_dict['values'] - if ('type' in result_dict and - result_dict['type'] == 'list_of_scalar_values' and values and - values != 'null'): + if ('type' in result_dict + and result_dict['type'] == 'list_of_scalar_values' and values + and values != 'null'): keyvals_dict[key] = sum(values) / float(len(values)) else: keyvals_dict[key] = values @@ -245,13 +245,14 @@ class Result(object): results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) - command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) + command = f'cp -r {self.results_dir}/* {self.temp_dir}' self.ce.RunCommand(command, print_to_console=False) command = ('./generate_test_report --no-color --csv %s' % (os.path.join('/tmp', os.path.basename(self.temp_dir)))) - _, out, _ = self.ce.ChrootRunCommandWOutput( - self.chromeos_root, command, print_to_console=False) + _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root, + command, + print_to_console=False) keyvals_dict = {} tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, self.temp_dir) @@ -322,8 +323,8 @@ class Result(object): idle_functions = { '[kernel.kallsyms]': - ('intel_idle', 'arch_cpu_idle', 'intel_idle', 'cpu_startup_entry', - 'default_idle', 'cpu_idle_loop', 'do_idle'), + ('intel_idle', 'arch_cpu_idle', 'intel_idle', 'cpu_startup_entry', + 'default_idle', 'cpu_idle_loop', 'do_idle'), } idle_samples = 0 @@ -390,8 +391,8 @@ class Result(object): result = ( self.FindFilesInResultsDir('-name histograms.json').splitlines()) else: - result = ( - self.FindFilesInResultsDir('-name results-chart.json').splitlines()) + result = (self.FindFilesInResultsDir( + '-name results-chart.json').splitlines()) return result def GetTurbostatFile(self): @@ -449,8 +450,8 @@ class Result(object): if debug_path: symfs = '--symfs ' + debug_path - vmlinux = '--vmlinux ' + os.path.join(debug_path, 'usr', 'lib', 'debug', - 'boot', 'vmlinux') + vmlinux = '--vmlinux ' + os.path.join(debug_path, 'usr', 'lib', + 'debug', 'boot', 'vmlinux') kallsyms = '' print('** WARNING **: --kallsyms option not applied, no System.map-* ' 'for downloaded image.') @@ -546,9 +547,9 @@ class Result(object): values = value_dict['values'] if not values: continue - if ('type' in value_dict and - value_dict['type'] == 'list_of_scalar_values' and - values != 'null'): + if ('type' in value_dict + and value_dict['type'] == 'list_of_scalar_values' + and values != 'null'): result = sum(values) / float(len(values)) else: result = values @@ -746,8 +747,9 @@ class Result(object): # order. heapq.heappush(cmd_top5_cpu_use[cmd_with_pid], round(cpu_use, 1)) - for consumer, usage in sorted( - cmd_total_cpu_use.items(), key=lambda x: x[1], reverse=True): + for consumer, usage in sorted(cmd_total_cpu_use.items(), + key=lambda x: x[1], + reverse=True): # Iterate through commands by descending order of total CPU usage. topcmd = { 'cmd': consumer, @@ -913,7 +915,8 @@ class Result(object): self.chromeos_root, path_str) if status: # Error of reading a perf.data profile is fatal. - raise PerfDataReadError(f'Failed to read perf.data profile: {path_str}') + raise PerfDataReadError( + f'Failed to read perf.data profile: {path_str}') # Pattern to search a line with "perf record" command line: # # cmdline : /usr/bin/perf record -e instructions -p 123" @@ -938,7 +941,8 @@ class Result(object): break else: # cmdline wasn't found in the header. It's a fatal error. - raise PerfDataReadError(f'Perf command line is not found in {path_str}') + raise PerfDataReadError( + f'Perf command line is not found in {path_str}') return pids def VerifyPerfDataPID(self): @@ -976,11 +980,11 @@ class Result(object): # Note that this function doesn't know anything about whether there is a # cache hit or miss. It should process results agnostic of the cache hit # state. - if (self.results_file and self.suite == 'telemetry_Crosperf' and - 'histograms.json' in self.results_file[0]): + if (self.results_file and self.suite == 'telemetry_Crosperf' + and 'histograms.json' in self.results_file[0]): self.keyvals = self.ProcessHistogramsResults() - elif (self.results_file and self.suite != 'telemetry_Crosperf' and - 'results-chart.json' in self.results_file[0]): + elif (self.results_file and self.suite != 'telemetry_Crosperf' + and 'results-chart.json' in self.results_file[0]): self.keyvals = self.ProcessChartResults() else: if not use_cache: @@ -1134,15 +1138,16 @@ class Result(object): f.write(machine_manager.machine_checksum_string[self.label.name]) if os.path.exists(cache_dir): - command = 'rm -rf {0}'.format(cache_dir) + command = f'rm -rf {cache_dir}' self.ce.RunCommand(command) - command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) - command += 'chmod g+x {0} && '.format(temp_dir) - command += 'mv {0} {1}'.format(temp_dir, cache_dir) + parent_dir = os.path.dirname(cache_dir) + command = f'mkdir -p {parent_dir} && ' + command += f'chmod g+x {temp_dir} && ' + command += f'mv {temp_dir} {cache_dir}' ret = self.ce.RunCommand(command) if ret: - command = 'rm -rf {0}'.format(temp_dir) + command = f'rm -rf {temp_dir}' self.ce.RunCommand(command) raise RuntimeError('Could not move dir %s to dir %s' % (temp_dir, cache_dir)) @@ -1241,8 +1246,8 @@ class TelemetryResult(Result): self.err = pickle.load(f) self.retval = pickle.load(f) - self.chrome_version = ( - super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir)) + self.chrome_version = (super(TelemetryResult, + self).GetChromeVersionFromCache(cache_dir)) self.ProcessResults() @@ -1304,10 +1309,10 @@ class ResultsCache(object): self.run_local = None self.cwp_dso = None - def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, - profiler_args, machine_manager, machine, board, cache_conditions, - logger_to_use, log_level, label, share_cache, suite, - show_all_results, run_local, cwp_dso): + def Init(self, chromeos_image, chromeos_root, test_name, iteration, + test_args, profiler_args, machine_manager, machine, board, + cache_conditions, logger_to_use, log_level, label, share_cache, + suite, show_all_results, run_local, cwp_dso): self.chromeos_image = chromeos_image self.chromeos_root = chromeos_root self.test_name = test_name @@ -1319,8 +1324,8 @@ class ResultsCache(object): self.machine_manager = machine_manager self.machine = machine self._logger = logger_to_use - self.ce = command_executer.GetCommandExecuter( - self._logger, log_level=log_level) + self.ce = command_executer.GetCommandExecuter(self._logger, + log_level=log_level) self.label = label self.share_cache = share_cache self.suite = suite @@ -1406,15 +1411,16 @@ class ResultsCache(object): temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, self.run_local) - test_args_checksum = hashlib.md5(temp_test_args.encode('utf-8')).hexdigest() + test_args_checksum = hashlib.md5( + temp_test_args.encode('utf-8')).hexdigest() return (image_path_checksum, self.test_name, str(self.iteration), - test_args_checksum, checksum, machine_checksum, machine_id_checksum, - str(self.CACHE_VERSION)) + test_args_checksum, checksum, machine_checksum, + machine_id_checksum, str(self.CACHE_VERSION)) def ReadResult(self): if CacheConditions.FALSE in self.cache_conditions: cache_dir = self.GetCacheDirForWrite() - command = 'rm -rf %s' % (cache_dir,) + command = 'rm -rf %s' % (cache_dir, ) self.ce.RunCommand(command) return None cache_dir = self.GetCacheDirForRead() @@ -1427,14 +1433,15 @@ class ResultsCache(object): if self.log_level == 'verbose': self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) - result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, - self.machine, cache_dir, self.test_name, - self.suite, self.cwp_dso) + result = Result.CreateFromCacheHit(self._logger, self.log_level, + self.label, self.machine, cache_dir, + self.test_name, self.suite, + self.cwp_dso) if not result: return None - if (result.retval == 0 or - CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): + if (result.retval == 0 + or CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): return result return None diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index df3a35e9..94b104c2 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -11,6 +11,7 @@ from __future__ import print_function import io import os +import pickle import shutil import tempfile import unittest @@ -31,6 +32,8 @@ from cros_utils import command_executer from cros_utils import logger from cros_utils import misc +# The following hardcoded string has blocked words replaced, and thus +# is not representative of a true crosperf output. # pylint: disable=line-too-long OUTPUT = """CMD (True): ./test_that.sh\ --remote=172.17.128.241 --board=lumpy LibCBench @@ -42,13 +45,13 @@ INFO : Running the following control files 1 times: INFO : * 'client/site_tests/platform_LibCBench/control' INFO : Running client test client/site_tests/platform_LibCBench/control -./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args +./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args ERROR:root:import statsd failed, no stats will be reported. 14:20:22 INFO | Results placed in /tmp/test_that.PO1234567/platform_LibCBench 14:20:22 INFO | Processing control file -14:20:23 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' +14:20:23 INFO | Starting main ssh connection '/usr/bin/ssh -a -x -N -o ControlMain=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-main/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 14:20:23 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. -14:20:23 INFO | INFO ---- ---- kernel=3.8.11 localtime=May 22 14:20:23 timestamp=1369257623 +14:20:23 INFO | INFO\t----\t----\tkernel=3.8.11\tlocaltime=May 22 14:20:23\ttimestamp=1369257623 14:20:23 INFO | Installing autotest on 172.17.128.241 14:20:23 INFO | Using installation dir /usr/local/autotest 14:20:23 WARNI| No job_repo_url for @@ -59,11 +62,11 @@ ERROR:root:import statsd failed, no stats will be reported. 14:20:24 INFO | Entered autotestd_monitor. 14:20:24 INFO | Finished launching tail subprocesses. 14:20:24 INFO | Finished waiting on autotestd to start. -14:20:26 INFO | START ---- ---- timestamp=1369257625 localtime=May 22 14:20:25 -14:20:26 INFO | START platform_LibCBench platform_LibCBench timestamp=1369257625 localtime=May 22 14:20:25 -14:20:30 INFO | GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 completed successfully -14:20:30 INFO | END GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 -14:20:31 INFO | END GOOD ---- ---- timestamp=1369257630 localtime=May 22 14:20:30 +14:20:26 INFO | START\t----\t----\ttimestamp=1369257625\tlocaltime=May 22 14:20:25 +14:20:26 INFO | \tSTART\tplatform_LibCBench\tplatform_LibCBench\ttimestamp=1369257625\tlocaltime=May 22 14:20:25 +14:20:30 INFO | \t\tGOOD\tplatform_LibCBench\tplatform_LibCBench\ttimestamp=1369257630\tlocaltime=May 22 14:20:30\tcompleted successfully +14:20:30 INFO | \tEND GOOD\tplatform_LibCBench\tplatform_LibCBench\ttimestamp=1369257630\tlocaltime=May 22 14:20:30 +14:20:31 INFO | END GOOD\t----\t----\ttimestamp=1369257630\tlocaltime=May 22 14:20:30 14:20:31 INFO | Got lock of exit_code_file. 14:20:31 INFO | Released lock of exit_code_file and closed it. OUTPUT: ============================== @@ -72,14 +75,14 @@ Done: 0% [ ] OUTPUT: Thread Status: RUNNING: 1 ('ttt: LibCBench (1)' 0:01:21) Machine Status: -Machine Thread Lock Status Checksum +Machine Thread Lock Status Checksum 172.17.128.241 ttt: LibCBench (1) True RUNNING 3ba9f2ecbb222f20887daea5583d86ba OUTPUT: ============================== 14:20:33 INFO | Killing child processes. 14:20:33 INFO | Client complete 14:20:33 INFO | Finished processing control file -14:20:33 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' +14:20:33 INFO | Starting main ssh connection '/usr/bin/ssh -a -x -N -o ControlMain=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-main/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 14:20:33 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. INFO : Test results: @@ -116,7 +119,7 @@ platform_LibCBench/platform_LibCBench b_utf8_onebyone__0_ ------------------------------------------------------------------- Total PASS: 2/2 (100%) -INFO : Elapsed time: 0m16s +INFO : Elapsed time: 0m16s """ error = """ @@ -484,8 +487,8 @@ class ResultTest(unittest.TestCase): def testCreateFromRun(self): result = MockResult.CreateFromRun(logger.GetLogger(), 'average', - self.mock_label, 'remote1', OUTPUT, error, - 0, True) + self.mock_label, 'remote1', OUTPUT, + error, 0, True) self.assertEqual(result.keyvals, keyvals) self.assertEqual(result.chroot_results_dir, '/tmp/test_that.PO1234567/platform_LibCBench') @@ -533,7 +536,8 @@ class ResultTest(unittest.TestCase): mock_runcmd.call_args_list[1]) self.assertEqual(mock_runcmd.call_args_list[0], mock_runcmd.call_args_list[2]) - self.assertEqual(mock_runcmd.call_args_list[0][0], ('mkdir -p /tmp/test',)) + self.assertEqual(mock_runcmd.call_args_list[0][0], + ('mkdir -p /tmp/test', )) # test 3. CopyFiles returns 1 (fails). mock_copyfiles.return_value = 1 @@ -715,7 +719,8 @@ class ResultTest(unittest.TestCase): mock_mkdtemp.return_value = TMP_DIR1 mock_chrootruncmd.return_value = [ - '', ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), '' + '', ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), + '' ] mock_getpath.return_value = TMP_DIR1 self.result.ce.ChrootRunCommandWOutput = mock_chrootruncmd @@ -730,7 +735,7 @@ class ResultTest(unittest.TestCase): self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) self.assertEqual(mock_runcmd.call_count, 1) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1,)) + ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1, )) self.assertEqual(mock_chrootruncmd.call_count, 1) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], @@ -770,7 +775,8 @@ class ResultTest(unittest.TestCase): @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommandWOutput') @mock.patch.object(os.path, 'exists') - def test_get_samples(self, mock_exists, mock_get_total_samples, mock_getpath): + def test_get_samples(self, mock_exists, mock_get_total_samples, + mock_getpath): self.result.perf_data_files = ['/tmp/results/perf.data'] self.result.board = 'samus' mock_getpath.return_value = '/usr/chromeos/chroot/tmp/results/perf.data' @@ -811,7 +817,7 @@ class ResultTest(unittest.TestCase): res = self.result.FindFilesInResultsDir('-name perf.data') self.assertEqual(mock_runcmd.call_count, 1) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('find /tmp/test_results -name perf.data',)) + ('find /tmp/test_results -name perf.data', )) self.assertEqual(res, '/tmp/test_results/perf.data') mock_runcmd.reset_mock() @@ -827,7 +833,8 @@ class ResultTest(unittest.TestCase): self.result.FindFilesInResultsDir = mock_findfiles res = self.result.GetPerfDataFiles() self.assertEqual(res, ['line1', 'line1']) - self.assertEqual(mock_findfiles.call_args_list[0][0], ('-name perf.data',)) + self.assertEqual(mock_findfiles.call_args_list[0][0], + ('-name perf.data', )) def test_get_perf_report_files(self): self.args = None @@ -958,16 +965,18 @@ class ResultTest(unittest.TestCase): """Verify perf PID which is present in TOP_DATA.""" self.result.top_cmds = TOP_DATA # pid is present in TOP_DATA. - with mock.patch.object( - Result, 'ReadPidFromPerfData', return_value=['5713']): + with mock.patch.object(Result, + 'ReadPidFromPerfData', + return_value=['5713']): self.result.VerifyPerfDataPID() def test_verify_perf_data_pid_fail(self): """Test perf PID missing in top raises the error.""" self.result.top_cmds = TOP_DATA # pid is not in the list of top processes. - with mock.patch.object( - Result, 'ReadPidFromPerfData', return_value=['9999']): + with mock.patch.object(Result, + 'ReadPidFromPerfData', + return_value=['9999']): with self.assertRaises(PidVerificationError): self.result.VerifyPerfDataPID() @@ -976,7 +985,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_ok(self, mock_runcmd): """Test perf header parser, normal flow.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] exp_pid = '12345' mock_runcmd.return_value = (0, PERF_DATA_HEADER.format(pid=exp_pid), '') pids = self.result.ReadPidFromPerfData() @@ -1007,7 +1018,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_no_pid(self, mock_runcmd): """Test perf.data without PID.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] cmd_line = '# cmdline : /usr/bin/perf record -e instructions' mock_runcmd.return_value = (0, cmd_line, '') pids = self.result.ReadPidFromPerfData() @@ -1019,7 +1032,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_system_wide(self, mock_runcmd): """Test reading from system-wide profile with PID.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] # There is '-p ' in command line but it's still system-wide: '-a'. cmd_line = '# cmdline : /usr/bin/perf record -e instructions -a -p 1234' mock_runcmd.return_value = (0, cmd_line, '') @@ -1032,7 +1047,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_read_fail(self, mock_runcmd): """Failure to read perf.data raises the error.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] # Error status of the profile read. mock_runcmd.return_value = (1, '', '') with self.assertRaises(PerfDataReadError): @@ -1043,7 +1060,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_fail(self, mock_runcmd): """Failure to find cmdline in perf.data header raises the error.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] # Empty output. mock_runcmd.return_value = (0, '', '') with self.assertRaises(PerfDataReadError): @@ -1273,7 +1292,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(misc, 'GetOutsideChrootPath') def test_populate_from_run(self, mock_getpath): - def FakeGetResultsDir(): self.callGetResultsDir = True return '/tmp/results_dir' @@ -1361,7 +1379,6 @@ class ResultTest(unittest.TestCase): return {'Total': 10} def test_process_results(self): - def FakeGatherPerfResults(): self.callGatherPerfResults = True @@ -1407,16 +1424,17 @@ class ResultTest(unittest.TestCase): self.result.ProcessResults() shutil.rmtree(os.path.dirname(self.result.results_file[0])) # Verify the summary for the story is correct - self.assertEqual(self.result.keyvals['timeToFirstContentfulPaint__typical'], - [880.000, u'ms_smallerIsBetter']) + self.assertEqual( + self.result.keyvals['timeToFirstContentfulPaint__typical'], + [880.000, u'ms_smallerIsBetter']) # Veirfy the summary for a certain stroy tag is correct self.assertEqual( - self.result - .keyvals['timeToFirstContentfulPaint__cache_temperature:cold'], + self.result. + keyvals['timeToFirstContentfulPaint__cache_temperature:cold'], [1000.000, u'ms_smallerIsBetter']) self.assertEqual( - self.result - .keyvals['timeToFirstContentfulPaint__cache_temperature:warm'], + self.result. + keyvals['timeToFirstContentfulPaint__cache_temperature:warm'], [800.000, u'ms_smallerIsBetter']) @mock.patch.object(Result, 'ProcessCpustatsResults') @@ -1572,7 +1590,8 @@ class ResultTest(unittest.TestCase): u'telemetry_page_measurement_results__num_errored': [0, u'count'], u'string-fasta__string-fasta': [23.2, u'ms'], u'crypto-sha1__crypto-sha1': [11.6, u'ms'], - u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], + u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': + [3.2, u'ms'], u'access-nsieve__access-nsieve': [7.9, u'ms'], u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], u'string-validate-input__string-validate-input': [19.3, u'ms'], @@ -1610,7 +1629,8 @@ class ResultTest(unittest.TestCase): u'telemetry_page_measurement_results__num_errored': [0, u'count'], u'string-fasta__string-fasta': [23.2, u'ms'], u'crypto-sha1__crypto-sha1': [11.6, u'ms'], - u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], + u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': + [3.2, u'ms'], u'access-nsieve__access-nsieve': [7.9, u'ms'], u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], u'string-validate-input__string-validate-input': [19.3, u'ms'], @@ -1657,8 +1677,9 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_getroot.call_count, 1) self.assertEqual(mock_runcmd.call_count, 2) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('rm -rf test_results_dir',)) - self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) + ('rm -rf test_results_dir', )) + self.assertEqual(mock_runcmd.call_args_list[1][0], + ('rm -rf testtemp_dir', )) # Test 2. Same, except ath results_dir name does not contain # 'test_that_results_' @@ -1672,8 +1693,9 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_getroot.call_count, 1) self.assertEqual(mock_runcmd.call_count, 2) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('rm -rf /tmp/tmp_AbcXyz',)) - self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) + ('rm -rf /tmp/tmp_AbcXyz', )) + self.assertEqual(mock_runcmd.call_args_list[1][0], + ('rm -rf testtemp_dir', )) # Test 3. mock_getroot returns nothing; 'rm_chroot_tmp' is False. mock_getroot.reset_mock() @@ -1681,7 +1703,8 @@ class ResultTest(unittest.TestCase): self.result.CleanUp(False) self.assertEqual(mock_getroot.call_count, 0) self.assertEqual(mock_runcmd.call_count, 1) - self.assertEqual(mock_runcmd.call_args_list[0][0], ('rm -rf testtemp_dir',)) + self.assertEqual(mock_runcmd.call_args_list[0][0], + ('rm -rf testtemp_dir', )) # Test 4. 'rm_chroot_tmp' is True, but result_dir & temp_dir are None. mock_getroot.reset_mock() @@ -1695,7 +1718,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(misc, 'GetInsideChrootPath') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') def test_store_to_cache_dir(self, mock_chrootruncmd, mock_getpath): - def FakeMkdtemp(directory=''): if directory: pass @@ -1730,7 +1752,7 @@ class ResultTest(unittest.TestCase): base_dir = os.path.join(os.getcwd(), 'test_cache/compare_output') self.assertTrue(os.path.exists(os.path.join(test_dir, 'autotest.tbz2'))) self.assertTrue(os.path.exists(os.path.join(test_dir, 'machine.txt'))) - self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.txt'))) + self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.pickle'))) f1 = os.path.join(test_dir, 'machine.txt') f2 = os.path.join(base_dir, 'machine.txt') @@ -1738,11 +1760,13 @@ class ResultTest(unittest.TestCase): [_, out, _] = self.result.ce.RunCommandWOutput(cmd) self.assertEqual(len(out), 0) - f1 = os.path.join(test_dir, 'results.txt') - f2 = os.path.join(base_dir, 'results.txt') - cmd = 'diff %s %s' % (f1, f2) - [_, out, _] = self.result.ce.RunCommandWOutput(cmd) - self.assertEqual(len(out), 0) + f1 = os.path.join(test_dir, 'results.pickle') + f2 = os.path.join(base_dir, 'results.pickle') + with open(f1, 'rb') as f: + f1_obj = pickle.load(f) + with open(f2, 'rb') as f: + f2_obj = pickle.load(f) + self.assertEqual(f1_obj, f2_obj) # Clean up after test. tempfile.mkdtemp = save_real_mkdtemp @@ -1753,87 +1777,87 @@ class ResultTest(unittest.TestCase): TELEMETRY_RESULT_KEYVALS = { 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'math-cordic (ms)': - '11.4', + '11.4', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-nbody (ms)': - '6.9', + '6.9', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-fannkuch (ms)': - '26.3', + '26.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'math-spectral-norm (ms)': - '6.3', + '6.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-nsieve-bits (ms)': - '9.3', + '9.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'math-partial-sums (ms)': - '32.8', + '32.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'regexp-dna (ms)': - '16.1', + '16.1', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' '3d-cube (ms)': - '42.7', + '42.7', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'crypto-md5 (ms)': - '10.8', + '10.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'crypto-sha1 (ms)': - '12.4', + '12.4', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-tagcloud (ms)': - '47.2', + '47.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-fasta (ms)': - '36.3', + '36.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-binary-trees (ms)': - '7.3', + '7.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'date-format-xparb (ms)': - '138.1', + '138.1', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'crypto-aes (ms)': - '19.2', + '19.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'Total (ms)': - '656.5', + '656.5', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-base64 (ms)': - '17.5', + '17.5', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-validate-input (ms)': - '24.8', + '24.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' '3d-raytrace (ms)': - '28.7', + '28.7', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'controlflow-recursive (ms)': - '5.3', + '5.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-bits-in-byte (ms)': - '9.8', + '9.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' '3d-morph (ms)': - '50.2', + '50.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-bitwise-and (ms)': - '8.8', + '8.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-nsieve (ms)': - '8.6', + '8.6', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'date-format-tofte (ms)': - '31.2', + '31.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-3bit-bits-in-byte (ms)': - '3.5', + '3.5', 'retval': - 0, + 0, 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-unpack-code (ms)': - '45.0' + '45.0' } PURE_TELEMETRY_OUTPUT = """ @@ -1854,12 +1878,10 @@ class TelemetryResultTest(unittest.TestCase): 'autotest_dir', 'debug_dir', '/tmp', 'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', False, None) - self.mock_machine = machine_manager.MockCrosMachine('falco.cros', - '/tmp/chromeos', - 'average') + self.mock_machine = machine_manager.MockCrosMachine( + 'falco.cros', '/tmp/chromeos', 'average') def test_populate_from_run(self): - def FakeProcessResults(): self.callFakeProcessResults = True @@ -1932,7 +1954,6 @@ class ResultsCacheTest(unittest.TestCase): @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') def test_get_cache_dir_for_write(self, mock_checksum): - def FakeGetMachines(label): if label: pass @@ -2041,7 +2062,8 @@ class ResultsCacheTest(unittest.TestCase): # Test 5. Generating cache name for writing, with local image type, and # specifying that the image path must match the cached image path. self.results_cache.label.image_type = 'local' - self.results_cache.cache_conditions.append(CacheConditions.IMAGE_PATH_MATCH) + self.results_cache.cache_conditions.append( + CacheConditions.IMAGE_PATH_MATCH) key_list = self.results_cache.GetCacheKeyList(False) self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') diff --git a/crosperf/test_cache/compare_output/results.pickle b/crosperf/test_cache/compare_output/results.pickle new file mode 100644 index 00000000..587863c5 Binary files /dev/null and b/crosperf/test_cache/compare_output/results.pickle differ diff --git a/crosperf/test_cache/compare_output/results.txt b/crosperf/test_cache/compare_output/results.txt deleted file mode 100644 index 592e7161..00000000 Binary files a/crosperf/test_cache/compare_output/results.txt and /dev/null differ diff --git a/crosperf/test_cache/test_input/results.pickle b/crosperf/test_cache/test_input/results.pickle new file mode 100644 index 00000000..33ba6ab7 --- /dev/null +++ b/crosperf/test_cache/test_input/results.pickle @@ -0,0 +1,6 @@ +S"11:22:08 INFO | Running autotest_quickmerge step.\n11:22:08 INFO | quickmerge| 11:22:08: INFO: RunCommand: sudo -- /usr/bin/python2.7 /mnt/host/source/chromite/bin/autotest_quickmerge '--board=lumpy'\n11:22:08 INFO | quickmerge| 11:22:08: INFO: RunCommand: find /build/lumpy/usr/local/build/autotest/ -path /build/lumpy/usr/local/build/autotest/ExternalSource -prune -o -path /build/lumpy/usr/local/build/autotest/logs -prune -o -path /build/lumpy/usr/local/build/autotest/results -prune -o -path /build/lumpy/usr/local/build/autotest/site-packages -prune -o -printf '%T@\\n'\n11:22:22 INFO | quickmerge| 11:22:22: INFO: RunCommand: find /mnt/host/source/src/third_party/autotest/files/ -path /mnt/host/source/src/third_party/autotest/files/ExternalSource -prune -o -path /mnt/host/source/src/third_party/autotest/files/logs -prune -o -path /mnt/host/source/src/third_party/autotest/files/results -prune -o -path /mnt/host/source/src/third_party/autotest/files/site-packages -prune -o -printf '%T@\\n'\n11:22:32 INFO | quickmerge| 11:22:32: INFO: The sysroot appears to be newer than the source tree, doing nothing and exiting now.\n11:22:32 INFO | Re-running test_that script in /build/lumpy/usr/local/build/autotest copy of autotest.\n11:22:33 INFO | Began logging to /tmp/test_that_results_zZZfQa\nAdding labels [u'cros-version:ad_hoc_build', u'board:lumpy'] to host chromeos2-row2-rack4-host11.cros\n13:22:33 INFO | Fetching suite for job named telemetry_Crosperf...\n13:22:43 INFO | Scheduling suite for job named telemetry_Crosperf...\n13:22:43 INFO | ... scheduled 1 job(s).\n13:22:43 INFO | autoserv| DEBUG:root:import statsd failed, no stats will be reported.\n13:22:43 INFO | autoserv| Results placed in /tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf\n13:22:43 INFO | autoserv| Logged pid 25397 to /tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf/.autoserv_execute\n13:22:43 INFO | autoserv| I am PID 25397\n13:22:43 INFO | autoserv| Not checking if job_repo_url contains autotest packages on ['chromeos2-row2-rack4-host11.cros']\n13:22:43 INFO | autoserv| Processing control file\n13:22:44 INFO | autoserv| START\ttelemetry_Crosperf\ttelemetry_Crosperf\ttimestamp=1401301364\tlocaltime=May 28 11:22:44\n13:22:44 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_HsB3vQssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/tmpxFy6lj -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=300 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n13:22:45 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_YTu9wYssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n13:22:45 INFO | autoserv| Installing autotest on chromeos2-row2-rack4-host11.cros\n13:22:45 INFO | autoserv| Using installation dir /tmp/sysinfo/autoserv-MxOMOw\n13:22:46 INFO | autoserv| No job_repo_url for \n13:22:46 INFO | autoserv| Could not install autotest using the packaging system: No repos to install an autotest client from. Trying other methods\n13:22:47 INFO | autoserv| Installation of autotest completed\n13:22:47 INFO | autoserv| Installing updated global_config.ini.\n13:22:48 INFO | autoserv| No job_repo_url for \n13:22:48 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:22:48 INFO | autoserv| Entered autotestd_monitor.\n13:22:48 INFO | autoserv| Finished launching tail subprocesses.\n13:22:48 INFO | autoserv| Finished waiting on autotestd to start.\n13:22:48 INFO | autoserv| START\t----\t----\ttimestamp=1401301368\tlocaltime=May 28 11:22:48\n13:22:48 INFO | autoserv| GOOD\t----\tsysinfo.before\ttimestamp=1401301368\tlocaltime=May 28 11:22:48\n13:22:48 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301368\tlocaltime=May 28 11:22:48\n13:22:48 INFO | autoserv| Got lock of exit_code_file.\n13:22:48 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:22:50 INFO | autoserv| Killing child processes.\n13:22:50 INFO | autoserv| Client complete\n13:22:52 INFO | autoserv| No job_repo_url for \n13:22:52 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:22:53 INFO | autoserv| Entered autotestd_monitor.\n13:22:53 INFO | autoserv| Finished launching tail subprocesses.\n13:22:53 INFO | autoserv| Finished waiting on autotestd to start.\n13:22:53 INFO | autoserv| START\t----\t----\ttimestamp=1401301373\tlocaltime=May 28 11:22:53\n13:22:53 INFO | autoserv| GOOD\t----\tsysinfo.iteration.before\ttimestamp=1401301373\tlocaltime=May 28 11:22:53\n13:22:53 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301373\tlocaltime=May 28 11:22:53\n13:22:53 INFO | autoserv| Got lock of exit_code_file.\n13:22:53 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:22:55 INFO | autoserv| Killing child processes.\n13:22:55 INFO | autoserv| Client complete\n13:22:55 INFO | autoserv| Using Chrome source tree at /tmp/chrome_root\n13:22:55 INFO | autoserv| CMD: /tmp/chrome_root/src/tools/perf/run_benchmark --browser=cros-chrome --remote=chromeos2-row2-rack4-host11.cros sunspider\n13:23:35 INFO | autoserv| Telemetry completed with exit code: 0.\n13:23:35 INFO | autoserv| stdout:Pages: [http___www.webkit.org_perf_sunspider-1.0.2_sunspider-1.0.2_driver.html]\n13:23:35 INFO | autoserv| RESULT 3d-cube: 3d-cube= [28,28,28,28,31,26,28,28,28,27] ms\n13:23:35 INFO | autoserv| Avg 3d-cube: 28.000000ms\n13:23:35 INFO | autoserv| Sd 3d-cube: 1.247219ms\n13:23:35 INFO | autoserv| RESULT 3d-morph: 3d-morph= [23,22,22,22,22,22,22,22,22,22] ms\n13:23:35 INFO | autoserv| Avg 3d-morph: 22.100000ms\n13:23:35 INFO | autoserv| Sd 3d-morph: 0.316228ms\n13:23:35 INFO | autoserv| RESULT 3d-raytrace: 3d-raytrace= [26,23,24,25,25,25,26,24,24,25] ms\n13:23:35 INFO | autoserv| Avg 3d-raytrace: 24.700000ms\n13:23:35 INFO | autoserv| Sd 3d-raytrace: 0.948683ms\n13:23:35 INFO | autoserv| *RESULT Total: Total= [443,440,440,447,451,435,441,449,449,445] ms\n13:23:35 INFO | autoserv| Avg Total: 444.000000ms\n13:23:35 INFO | autoserv| Sd Total: 5.077182ms\n13:23:35 INFO | autoserv| RESULT access-binary-trees: access-binary-trees= [4,3,5,6,5,5,3,5,5,4] ms\n13:23:35 INFO | autoserv| Avg access-binary-trees: 4.500000ms\n13:23:35 INFO | autoserv| Sd access-binary-trees: 0.971825ms\n13:23:35 INFO | autoserv| RESULT access-fannkuch: access-fannkuch= [19,18,17,18,17,18,18,18,17,18] ms\n13:23:35 INFO | autoserv| Avg access-fannkuch: 17.800000ms\n13:23:35 INFO | autoserv| Sd access-fannkuch: 0.632456ms\n13:23:35 INFO | autoserv| RESULT access-nbody: access-nbody= [7,9,8,7,12,8,9,10,8,7] ms\n13:23:35 INFO | autoserv| Avg access-nbody: 8.500000ms\n13:23:35 INFO | autoserv| Sd access-nbody: 1.581139ms\n13:23:35 INFO | autoserv| RESULT access-nsieve: access-nsieve= [9,8,8,8,8,7,8,7,8,8] ms\n13:23:35 INFO | autoserv| Avg access-nsieve: 7.900000ms\n13:23:35 INFO | autoserv| Sd access-nsieve: 0.567646ms\n13:23:35 INFO | autoserv| RESULT bitops-3bit-bits-in-byte: bitops-3bit-bits-in-byte= [3,3,3,3,3,3,3,4,4,3] ms\n13:23:35 INFO | autoserv| Avg bitops-3bit-bits-in-byte: 3.200000ms\n13:23:35 INFO | autoserv| Sd bitops-3bit-bits-in-byte: 0.421637ms\n13:23:35 INFO | autoserv| RESULT bitops-bits-in-byte: bitops-bits-in-byte= [9,9,9,9,9,9,9,9,9,10] ms\n13:23:35 INFO | autoserv| Avg bitops-bits-in-byte: 9.100000ms\n13:23:35 INFO | autoserv| Sd bitops-bits-in-byte: 0.316228ms\n13:23:35 INFO | autoserv| RESULT bitops-bitwise-and: bitops-bitwise-and= [8,8,7,9,8,9,8,8,9,10] ms\n13:23:35 INFO | autoserv| Avg bitops-bitwise-and: 8.400000ms\n13:23:35 INFO | autoserv| Sd bitops-bitwise-and: 0.843274ms\n13:23:35 INFO | autoserv| RESULT bitops-nsieve-bits: bitops-nsieve-bits= [9,9,9,9,9,9,9,11,11,9] ms\n13:23:35 INFO | autoserv| Avg bitops-nsieve-bits: 9.400000ms\n13:23:35 INFO | autoserv| Sd bitops-nsieve-bits: 0.843274ms\n13:23:35 INFO | autoserv| RESULT controlflow-recursive: controlflow-recursive= [5,5,5,4,4,4,5,4,4,4] ms\n13:23:35 INFO | autoserv| Avg controlflow-recursive: 4.400000ms\n13:23:35 INFO | autoserv| Sd controlflow-recursive: 0.516398ms\n13:23:35 INFO | autoserv| RESULT crypto-aes: crypto-aes= [14,16,15,16,15,14,17,14,15,16] ms\n13:23:35 INFO | autoserv| Avg crypto-aes: 15.200000ms\n13:23:35 INFO | autoserv| Sd crypto-aes: 1.032796ms\n13:23:35 INFO | autoserv| RESULT crypto-md5: crypto-md5= [10,11,11,11,10,10,11,10,10,11] ms\n13:23:35 INFO | autoserv| Avg crypto-md5: 10.500000ms\n13:23:35 INFO | autoserv| Sd crypto-md5: 0.527046ms\n13:23:35 INFO | autoserv| RESULT crypto-sha1: crypto-sha1= [11,11,12,12,12,12,12,10,13,11] ms\n13:23:35 INFO | autoserv| Avg crypto-sha1: 11.600000ms\n13:23:35 INFO | autoserv| Sd crypto-sha1: 0.843274ms\n13:23:35 INFO | autoserv| RESULT date-format-tofte: date-format-tofte= [28,25,25,26,26,27,26,28,27,25] ms\n13:23:35 INFO | autoserv| Avg date-format-tofte: 26.300000ms\n13:23:35 INFO | autoserv| Sd date-format-tofte: 1.159502ms\n13:23:35 INFO | autoserv| RESULT date-format-xparb: date-format-xparb= [21,22,21,21,21,20,20,20,21,22] ms\n13:23:35 INFO | autoserv| Avg date-format-xparb: 20.900000ms\n13:23:35 INFO | autoserv| Sd date-format-xparb: 0.737865ms\n13:23:35 INFO | autoserv| RESULT math-cordic: math-cordic= [8,8,8,9,9,9,9,9,9,9] ms\n13:23:35 INFO | autoserv| Avg math-cordic: 8.700000ms\n13:23:35 INFO | autoserv| Sd math-cordic: 0.483046ms\n13:23:35 INFO | autoserv| RESULT math-partial-sums: math-partial-sums= [22,22,22,21,23,20,20,23,25,22] ms\n13:23:35 INFO | autoserv| Avg math-partial-sums: 22.000000ms\n13:23:35 INFO | autoserv| Sd math-partial-sums: 1.490712ms\n13:23:35 INFO | autoserv| RESULT math-spectral-norm: math-spectral-norm= [6,7,6,7,7,6,7,6,7,7] ms\n13:23:35 INFO | autoserv| Avg math-spectral-norm: 6.600000ms\n13:23:35 INFO | autoserv| Sd math-spectral-norm: 0.516398ms\n13:23:35 INFO | autoserv| RESULT regexp-dna: regexp-dna= [16,16,17,16,16,16,16,16,17,16] ms\n13:23:35 INFO | autoserv| Avg regexp-dna: 16.200000ms\n13:23:35 INFO | autoserv| Sd regexp-dna: 0.421637ms\n13:23:35 INFO | autoserv| RESULT string-base64: string-base64= [17,16,16,16,17,16,16,16,14,16] ms\n13:23:35 INFO | autoserv| Avg string-base64: 16.000000ms\n13:23:35 INFO | autoserv| Sd string-base64: 0.816497ms\n13:23:35 INFO | autoserv| RESULT string-fasta: string-fasta= [23,22,23,24,23,23,23,25,23,23] ms\n13:23:35 INFO | autoserv| Avg string-fasta: 23.200000ms\n13:23:35 INFO | autoserv| Sd string-fasta: 0.788811ms\n13:23:35 INFO | autoserv| RESULT string-tagcloud: string-tagcloud= [53,52,54,53,53,52,51,54,53,53] ms\n13:23:35 INFO | autoserv| Avg string-tagcloud: 52.800000ms\n13:23:35 INFO | autoserv| Sd string-tagcloud: 0.918937ms\n13:23:35 INFO | autoserv| RESULT string-unpack-code: string-unpack-code= [46,47,46,48,47,46,46,47,47,47] ms\n13:23:35 INFO | autoserv| Avg string-unpack-code: 46.700000ms\n13:23:35 INFO | autoserv| Sd string-unpack-code: 0.674949ms\n13:23:35 INFO | autoserv| RESULT string-validate-input: string-validate-input= [18,20,19,19,19,19,19,21,19,20] ms\n13:23:35 INFO | autoserv| Avg string-validate-input: 19.300000ms\n13:23:35 INFO | autoserv| Sd string-validate-input: 0.823273ms\n13:23:35 INFO | autoserv| RESULT telemetry_page_measurement_results: num_failed= 0 count\n13:23:35 INFO | autoserv| RESULT telemetry_page_measurement_results: num_errored= 0 count\n13:23:35 INFO | autoserv| \n13:23:35 INFO | autoserv| View result at file:///tmp/chrome_root/src/tools/perf/results.html\n13:23:35 INFO | autoserv| \n13:23:35 INFO | autoserv| stderr:\n13:23:35 INFO | autoserv| No job_repo_url for \n13:23:35 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:23:36 INFO | autoserv| Entered autotestd_monitor.\n13:23:36 INFO | autoserv| Finished launching tail subprocesses.\n13:23:36 INFO | autoserv| Finished waiting on autotestd to start.\n13:23:37 INFO | autoserv| START\t----\t----\ttimestamp=1401301417\tlocaltime=May 28 11:23:37\n13:23:37 INFO | autoserv| GOOD\t----\tsysinfo.iteration.after\ttimestamp=1401301417\tlocaltime=May 28 11:23:37\n13:23:37 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301417\tlocaltime=May 28 11:23:37\n13:23:37 INFO | autoserv| Got lock of exit_code_file.\n13:23:37 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:23:39 INFO | autoserv| Killing child processes.\n13:23:39 INFO | autoserv| Client complete\n13:23:39 INFO | autoserv| No job_repo_url for \n13:23:40 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:23:40 INFO | autoserv| Entered autotestd_monitor.\n13:23:40 INFO | autoserv| Finished launching tail subprocesses.\n13:23:40 INFO | autoserv| Finished waiting on autotestd to start.\n13:23:40 INFO | autoserv| START\t----\t----\ttimestamp=1401301420\tlocaltime=May 28 11:23:40\n13:23:40 INFO | autoserv| GOOD\t----\tsysinfo.after\ttimestamp=1401301420\tlocaltime=May 28 11:23:40\n13:23:40 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301420\tlocaltime=May 28 11:23:40\n13:23:40 INFO | autoserv| Got lock of exit_code_file.\n13:23:40 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:23:42 INFO | autoserv| Killing child processes.\n13:23:42 INFO | autoserv| Client complete\n13:23:44 INFO | autoserv| GOOD\ttelemetry_Crosperf\ttelemetry_Crosperf\ttimestamp=1401301424\tlocaltime=May 28 11:23:44\tcompleted successfully\n13:23:44 INFO | autoserv| END GOOD\ttelemetry_Crosperf\ttelemetry_Crosperf\ttimestamp=1401301424\tlocaltime=May 28 11:23:44\n13:23:44 INFO | autoserv| Finished processing control file\n13:23:44 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_UyjlWMssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/tmpCvMigR -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=300 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n13:23:45 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_w_KGTassh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n-----------------------------------------------------------------------------------------\n/tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf [ PASSED ]\n/tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf/telemetry_Crosperf [ PASSED ]\n-----------------------------------------------------------------------------------------\nTotal PASS: 2/2 (100%)\n\n13:23:47 INFO | Finished running tests. Results can be found in /tmp/test_that_results_zZZfQa\n" +p0 +.S'INFO:root:Identity added: /tmp/test_that_results_PPRMIh/testing_rsa (/tmp/test_that_results_PPRMIh/testing_rsa)\nINFO:root:Identity added: /tmp/test_that_results_zZZfQa/testing_rsa (/tmp/test_that_results_zZZfQa/testing_rsa)\n' +p0 +.I0 +. \ No newline at end of file diff --git a/crosperf/test_cache/test_input/results.txt b/crosperf/test_cache/test_input/results.txt deleted file mode 100644 index 33ba6ab7..00000000 --- a/crosperf/test_cache/test_input/results.txt +++ /dev/null @@ -1,6 +0,0 @@ -S"11:22:08 INFO | Running autotest_quickmerge step.\n11:22:08 INFO | quickmerge| 11:22:08: INFO: RunCommand: sudo -- /usr/bin/python2.7 /mnt/host/source/chromite/bin/autotest_quickmerge '--board=lumpy'\n11:22:08 INFO | quickmerge| 11:22:08: INFO: RunCommand: find /build/lumpy/usr/local/build/autotest/ -path /build/lumpy/usr/local/build/autotest/ExternalSource -prune -o -path /build/lumpy/usr/local/build/autotest/logs -prune -o -path /build/lumpy/usr/local/build/autotest/results -prune -o -path /build/lumpy/usr/local/build/autotest/site-packages -prune -o -printf '%T@\\n'\n11:22:22 INFO | quickmerge| 11:22:22: INFO: RunCommand: find /mnt/host/source/src/third_party/autotest/files/ -path /mnt/host/source/src/third_party/autotest/files/ExternalSource -prune -o -path /mnt/host/source/src/third_party/autotest/files/logs -prune -o -path /mnt/host/source/src/third_party/autotest/files/results -prune -o -path /mnt/host/source/src/third_party/autotest/files/site-packages -prune -o -printf '%T@\\n'\n11:22:32 INFO | quickmerge| 11:22:32: INFO: The sysroot appears to be newer than the source tree, doing nothing and exiting now.\n11:22:32 INFO | Re-running test_that script in /build/lumpy/usr/local/build/autotest copy of autotest.\n11:22:33 INFO | Began logging to /tmp/test_that_results_zZZfQa\nAdding labels [u'cros-version:ad_hoc_build', u'board:lumpy'] to host chromeos2-row2-rack4-host11.cros\n13:22:33 INFO | Fetching suite for job named telemetry_Crosperf...\n13:22:43 INFO | Scheduling suite for job named telemetry_Crosperf...\n13:22:43 INFO | ... scheduled 1 job(s).\n13:22:43 INFO | autoserv| DEBUG:root:import statsd failed, no stats will be reported.\n13:22:43 INFO | autoserv| Results placed in /tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf\n13:22:43 INFO | autoserv| Logged pid 25397 to /tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf/.autoserv_execute\n13:22:43 INFO | autoserv| I am PID 25397\n13:22:43 INFO | autoserv| Not checking if job_repo_url contains autotest packages on ['chromeos2-row2-rack4-host11.cros']\n13:22:43 INFO | autoserv| Processing control file\n13:22:44 INFO | autoserv| START\ttelemetry_Crosperf\ttelemetry_Crosperf\ttimestamp=1401301364\tlocaltime=May 28 11:22:44\n13:22:44 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_HsB3vQssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/tmpxFy6lj -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=300 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n13:22:45 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_YTu9wYssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n13:22:45 INFO | autoserv| Installing autotest on chromeos2-row2-rack4-host11.cros\n13:22:45 INFO | autoserv| Using installation dir /tmp/sysinfo/autoserv-MxOMOw\n13:22:46 INFO | autoserv| No job_repo_url for \n13:22:46 INFO | autoserv| Could not install autotest using the packaging system: No repos to install an autotest client from. Trying other methods\n13:22:47 INFO | autoserv| Installation of autotest completed\n13:22:47 INFO | autoserv| Installing updated global_config.ini.\n13:22:48 INFO | autoserv| No job_repo_url for \n13:22:48 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:22:48 INFO | autoserv| Entered autotestd_monitor.\n13:22:48 INFO | autoserv| Finished launching tail subprocesses.\n13:22:48 INFO | autoserv| Finished waiting on autotestd to start.\n13:22:48 INFO | autoserv| START\t----\t----\ttimestamp=1401301368\tlocaltime=May 28 11:22:48\n13:22:48 INFO | autoserv| GOOD\t----\tsysinfo.before\ttimestamp=1401301368\tlocaltime=May 28 11:22:48\n13:22:48 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301368\tlocaltime=May 28 11:22:48\n13:22:48 INFO | autoserv| Got lock of exit_code_file.\n13:22:48 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:22:50 INFO | autoserv| Killing child processes.\n13:22:50 INFO | autoserv| Client complete\n13:22:52 INFO | autoserv| No job_repo_url for \n13:22:52 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:22:53 INFO | autoserv| Entered autotestd_monitor.\n13:22:53 INFO | autoserv| Finished launching tail subprocesses.\n13:22:53 INFO | autoserv| Finished waiting on autotestd to start.\n13:22:53 INFO | autoserv| START\t----\t----\ttimestamp=1401301373\tlocaltime=May 28 11:22:53\n13:22:53 INFO | autoserv| GOOD\t----\tsysinfo.iteration.before\ttimestamp=1401301373\tlocaltime=May 28 11:22:53\n13:22:53 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301373\tlocaltime=May 28 11:22:53\n13:22:53 INFO | autoserv| Got lock of exit_code_file.\n13:22:53 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:22:55 INFO | autoserv| Killing child processes.\n13:22:55 INFO | autoserv| Client complete\n13:22:55 INFO | autoserv| Using Chrome source tree at /tmp/chrome_root\n13:22:55 INFO | autoserv| CMD: /tmp/chrome_root/src/tools/perf/run_benchmark --browser=cros-chrome --remote=chromeos2-row2-rack4-host11.cros sunspider\n13:23:35 INFO | autoserv| Telemetry completed with exit code: 0.\n13:23:35 INFO | autoserv| stdout:Pages: [http___www.webkit.org_perf_sunspider-1.0.2_sunspider-1.0.2_driver.html]\n13:23:35 INFO | autoserv| RESULT 3d-cube: 3d-cube= [28,28,28,28,31,26,28,28,28,27] ms\n13:23:35 INFO | autoserv| Avg 3d-cube: 28.000000ms\n13:23:35 INFO | autoserv| Sd 3d-cube: 1.247219ms\n13:23:35 INFO | autoserv| RESULT 3d-morph: 3d-morph= [23,22,22,22,22,22,22,22,22,22] ms\n13:23:35 INFO | autoserv| Avg 3d-morph: 22.100000ms\n13:23:35 INFO | autoserv| Sd 3d-morph: 0.316228ms\n13:23:35 INFO | autoserv| RESULT 3d-raytrace: 3d-raytrace= [26,23,24,25,25,25,26,24,24,25] ms\n13:23:35 INFO | autoserv| Avg 3d-raytrace: 24.700000ms\n13:23:35 INFO | autoserv| Sd 3d-raytrace: 0.948683ms\n13:23:35 INFO | autoserv| *RESULT Total: Total= [443,440,440,447,451,435,441,449,449,445] ms\n13:23:35 INFO | autoserv| Avg Total: 444.000000ms\n13:23:35 INFO | autoserv| Sd Total: 5.077182ms\n13:23:35 INFO | autoserv| RESULT access-binary-trees: access-binary-trees= [4,3,5,6,5,5,3,5,5,4] ms\n13:23:35 INFO | autoserv| Avg access-binary-trees: 4.500000ms\n13:23:35 INFO | autoserv| Sd access-binary-trees: 0.971825ms\n13:23:35 INFO | autoserv| RESULT access-fannkuch: access-fannkuch= [19,18,17,18,17,18,18,18,17,18] ms\n13:23:35 INFO | autoserv| Avg access-fannkuch: 17.800000ms\n13:23:35 INFO | autoserv| Sd access-fannkuch: 0.632456ms\n13:23:35 INFO | autoserv| RESULT access-nbody: access-nbody= [7,9,8,7,12,8,9,10,8,7] ms\n13:23:35 INFO | autoserv| Avg access-nbody: 8.500000ms\n13:23:35 INFO | autoserv| Sd access-nbody: 1.581139ms\n13:23:35 INFO | autoserv| RESULT access-nsieve: access-nsieve= [9,8,8,8,8,7,8,7,8,8] ms\n13:23:35 INFO | autoserv| Avg access-nsieve: 7.900000ms\n13:23:35 INFO | autoserv| Sd access-nsieve: 0.567646ms\n13:23:35 INFO | autoserv| RESULT bitops-3bit-bits-in-byte: bitops-3bit-bits-in-byte= [3,3,3,3,3,3,3,4,4,3] ms\n13:23:35 INFO | autoserv| Avg bitops-3bit-bits-in-byte: 3.200000ms\n13:23:35 INFO | autoserv| Sd bitops-3bit-bits-in-byte: 0.421637ms\n13:23:35 INFO | autoserv| RESULT bitops-bits-in-byte: bitops-bits-in-byte= [9,9,9,9,9,9,9,9,9,10] ms\n13:23:35 INFO | autoserv| Avg bitops-bits-in-byte: 9.100000ms\n13:23:35 INFO | autoserv| Sd bitops-bits-in-byte: 0.316228ms\n13:23:35 INFO | autoserv| RESULT bitops-bitwise-and: bitops-bitwise-and= [8,8,7,9,8,9,8,8,9,10] ms\n13:23:35 INFO | autoserv| Avg bitops-bitwise-and: 8.400000ms\n13:23:35 INFO | autoserv| Sd bitops-bitwise-and: 0.843274ms\n13:23:35 INFO | autoserv| RESULT bitops-nsieve-bits: bitops-nsieve-bits= [9,9,9,9,9,9,9,11,11,9] ms\n13:23:35 INFO | autoserv| Avg bitops-nsieve-bits: 9.400000ms\n13:23:35 INFO | autoserv| Sd bitops-nsieve-bits: 0.843274ms\n13:23:35 INFO | autoserv| RESULT controlflow-recursive: controlflow-recursive= [5,5,5,4,4,4,5,4,4,4] ms\n13:23:35 INFO | autoserv| Avg controlflow-recursive: 4.400000ms\n13:23:35 INFO | autoserv| Sd controlflow-recursive: 0.516398ms\n13:23:35 INFO | autoserv| RESULT crypto-aes: crypto-aes= [14,16,15,16,15,14,17,14,15,16] ms\n13:23:35 INFO | autoserv| Avg crypto-aes: 15.200000ms\n13:23:35 INFO | autoserv| Sd crypto-aes: 1.032796ms\n13:23:35 INFO | autoserv| RESULT crypto-md5: crypto-md5= [10,11,11,11,10,10,11,10,10,11] ms\n13:23:35 INFO | autoserv| Avg crypto-md5: 10.500000ms\n13:23:35 INFO | autoserv| Sd crypto-md5: 0.527046ms\n13:23:35 INFO | autoserv| RESULT crypto-sha1: crypto-sha1= [11,11,12,12,12,12,12,10,13,11] ms\n13:23:35 INFO | autoserv| Avg crypto-sha1: 11.600000ms\n13:23:35 INFO | autoserv| Sd crypto-sha1: 0.843274ms\n13:23:35 INFO | autoserv| RESULT date-format-tofte: date-format-tofte= [28,25,25,26,26,27,26,28,27,25] ms\n13:23:35 INFO | autoserv| Avg date-format-tofte: 26.300000ms\n13:23:35 INFO | autoserv| Sd date-format-tofte: 1.159502ms\n13:23:35 INFO | autoserv| RESULT date-format-xparb: date-format-xparb= [21,22,21,21,21,20,20,20,21,22] ms\n13:23:35 INFO | autoserv| Avg date-format-xparb: 20.900000ms\n13:23:35 INFO | autoserv| Sd date-format-xparb: 0.737865ms\n13:23:35 INFO | autoserv| RESULT math-cordic: math-cordic= [8,8,8,9,9,9,9,9,9,9] ms\n13:23:35 INFO | autoserv| Avg math-cordic: 8.700000ms\n13:23:35 INFO | autoserv| Sd math-cordic: 0.483046ms\n13:23:35 INFO | autoserv| RESULT math-partial-sums: math-partial-sums= [22,22,22,21,23,20,20,23,25,22] ms\n13:23:35 INFO | autoserv| Avg math-partial-sums: 22.000000ms\n13:23:35 INFO | autoserv| Sd math-partial-sums: 1.490712ms\n13:23:35 INFO | autoserv| RESULT math-spectral-norm: math-spectral-norm= [6,7,6,7,7,6,7,6,7,7] ms\n13:23:35 INFO | autoserv| Avg math-spectral-norm: 6.600000ms\n13:23:35 INFO | autoserv| Sd math-spectral-norm: 0.516398ms\n13:23:35 INFO | autoserv| RESULT regexp-dna: regexp-dna= [16,16,17,16,16,16,16,16,17,16] ms\n13:23:35 INFO | autoserv| Avg regexp-dna: 16.200000ms\n13:23:35 INFO | autoserv| Sd regexp-dna: 0.421637ms\n13:23:35 INFO | autoserv| RESULT string-base64: string-base64= [17,16,16,16,17,16,16,16,14,16] ms\n13:23:35 INFO | autoserv| Avg string-base64: 16.000000ms\n13:23:35 INFO | autoserv| Sd string-base64: 0.816497ms\n13:23:35 INFO | autoserv| RESULT string-fasta: string-fasta= [23,22,23,24,23,23,23,25,23,23] ms\n13:23:35 INFO | autoserv| Avg string-fasta: 23.200000ms\n13:23:35 INFO | autoserv| Sd string-fasta: 0.788811ms\n13:23:35 INFO | autoserv| RESULT string-tagcloud: string-tagcloud= [53,52,54,53,53,52,51,54,53,53] ms\n13:23:35 INFO | autoserv| Avg string-tagcloud: 52.800000ms\n13:23:35 INFO | autoserv| Sd string-tagcloud: 0.918937ms\n13:23:35 INFO | autoserv| RESULT string-unpack-code: string-unpack-code= [46,47,46,48,47,46,46,47,47,47] ms\n13:23:35 INFO | autoserv| Avg string-unpack-code: 46.700000ms\n13:23:35 INFO | autoserv| Sd string-unpack-code: 0.674949ms\n13:23:35 INFO | autoserv| RESULT string-validate-input: string-validate-input= [18,20,19,19,19,19,19,21,19,20] ms\n13:23:35 INFO | autoserv| Avg string-validate-input: 19.300000ms\n13:23:35 INFO | autoserv| Sd string-validate-input: 0.823273ms\n13:23:35 INFO | autoserv| RESULT telemetry_page_measurement_results: num_failed= 0 count\n13:23:35 INFO | autoserv| RESULT telemetry_page_measurement_results: num_errored= 0 count\n13:23:35 INFO | autoserv| \n13:23:35 INFO | autoserv| View result at file:///tmp/chrome_root/src/tools/perf/results.html\n13:23:35 INFO | autoserv| \n13:23:35 INFO | autoserv| stderr:\n13:23:35 INFO | autoserv| No job_repo_url for \n13:23:35 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:23:36 INFO | autoserv| Entered autotestd_monitor.\n13:23:36 INFO | autoserv| Finished launching tail subprocesses.\n13:23:36 INFO | autoserv| Finished waiting on autotestd to start.\n13:23:37 INFO | autoserv| START\t----\t----\ttimestamp=1401301417\tlocaltime=May 28 11:23:37\n13:23:37 INFO | autoserv| GOOD\t----\tsysinfo.iteration.after\ttimestamp=1401301417\tlocaltime=May 28 11:23:37\n13:23:37 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301417\tlocaltime=May 28 11:23:37\n13:23:37 INFO | autoserv| Got lock of exit_code_file.\n13:23:37 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:23:39 INFO | autoserv| Killing child processes.\n13:23:39 INFO | autoserv| Client complete\n13:23:39 INFO | autoserv| No job_repo_url for \n13:23:40 INFO | autoserv| Executing /tmp/sysinfo/autoserv-MxOMOw/bin/autotest /tmp/sysinfo/autoserv-MxOMOw/control phase 0\n13:23:40 INFO | autoserv| Entered autotestd_monitor.\n13:23:40 INFO | autoserv| Finished launching tail subprocesses.\n13:23:40 INFO | autoserv| Finished waiting on autotestd to start.\n13:23:40 INFO | autoserv| START\t----\t----\ttimestamp=1401301420\tlocaltime=May 28 11:23:40\n13:23:40 INFO | autoserv| GOOD\t----\tsysinfo.after\ttimestamp=1401301420\tlocaltime=May 28 11:23:40\n13:23:40 INFO | autoserv| END GOOD\t----\t----\ttimestamp=1401301420\tlocaltime=May 28 11:23:40\n13:23:40 INFO | autoserv| Got lock of exit_code_file.\n13:23:40 INFO | autoserv| Released lock of exit_code_file and closed it.\n13:23:42 INFO | autoserv| Killing child processes.\n13:23:42 INFO | autoserv| Client complete\n13:23:44 INFO | autoserv| GOOD\ttelemetry_Crosperf\ttelemetry_Crosperf\ttimestamp=1401301424\tlocaltime=May 28 11:23:44\tcompleted successfully\n13:23:44 INFO | autoserv| END GOOD\ttelemetry_Crosperf\ttelemetry_Crosperf\ttimestamp=1401301424\tlocaltime=May 28 11:23:44\n13:23:44 INFO | autoserv| Finished processing control file\n13:23:44 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_UyjlWMssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/tmpCvMigR -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=300 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n13:23:45 INFO | autoserv| Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_w_KGTassh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 chromeos2-row2-rack4-host11.cros'\n-----------------------------------------------------------------------------------------\n/tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf [ PASSED ]\n/tmp/test_that_results_zZZfQa/results-1-telemetry_Crosperf/telemetry_Crosperf [ PASSED ]\n-----------------------------------------------------------------------------------------\nTotal PASS: 2/2 (100%)\n\n13:23:47 INFO | Finished running tests. Results can be found in /tmp/test_that_results_zZZfQa\n" -p0 -.S'INFO:root:Identity added: /tmp/test_that_results_PPRMIh/testing_rsa (/tmp/test_that_results_PPRMIh/testing_rsa)\nINFO:root:Identity added: /tmp/test_that_results_zZZfQa/testing_rsa (/tmp/test_that_results_zZZfQa/testing_rsa)\n' -p0 -.I0 -. \ No newline at end of file diff --git a/crosperf/test_cache/test_puretelemetry_input/results.pickle b/crosperf/test_cache/test_puretelemetry_input/results.pickle new file mode 100644 index 00000000..497d1cf3 --- /dev/null +++ b/crosperf/test_cache/test_puretelemetry_input/results.pickle @@ -0,0 +1,6 @@ +S'page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-trees (ms),access-fannkuch (ms),access-nbody (ms),access-nsieve (ms),bitops-3bit-bits-in-byte (ms),bitops-bits-in-byte (ms),bitops-bitwise-and (ms),bitops-nsieve-bits (ms),controlflow-recursive (ms),crypto-aes (ms),crypto-md5 (ms),crypto-sha1 (ms),date-format-tofte (ms),date-format-xparb (ms),math-cordic (ms),math-partial-sums (ms),math-spectral-norm (ms),regexp-dna (ms),string-base64 (ms),string-fasta (ms),string-tagcloud (ms),string-unpack-code (ms),string-validate-input (ms)\r\nhttp://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html,42.7,50.2,28.7,656.5,7.3,26.3,6.9,8.6,3.5,9.8,8.8,9.3,5.3,19.2,10.8,12.4,31.2,138.1,11.4,32.8,6.3,16.1,17.5,36.3,47.2,45.0,24.8\r\n' +p0 +.S'' +p0 +.I0 +. \ No newline at end of file diff --git a/crosperf/test_cache/test_puretelemetry_input/results.txt b/crosperf/test_cache/test_puretelemetry_input/results.txt deleted file mode 100644 index 497d1cf3..00000000 --- a/crosperf/test_cache/test_puretelemetry_input/results.txt +++ /dev/null @@ -1,6 +0,0 @@ -S'page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-trees (ms),access-fannkuch (ms),access-nbody (ms),access-nsieve (ms),bitops-3bit-bits-in-byte (ms),bitops-bits-in-byte (ms),bitops-bitwise-and (ms),bitops-nsieve-bits (ms),controlflow-recursive (ms),crypto-aes (ms),crypto-md5 (ms),crypto-sha1 (ms),date-format-tofte (ms),date-format-xparb (ms),math-cordic (ms),math-partial-sums (ms),math-spectral-norm (ms),regexp-dna (ms),string-base64 (ms),string-fasta (ms),string-tagcloud (ms),string-unpack-code (ms),string-validate-input (ms)\r\nhttp://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html,42.7,50.2,28.7,656.5,7.3,26.3,6.9,8.6,3.5,9.8,8.8,9.3,5.3,19.2,10.8,12.4,31.2,138.1,11.4,32.8,6.3,16.1,17.5,36.3,47.2,45.0,24.8\r\n' -p0 -.S'' -p0 -.I0 -. \ No newline at end of file diff --git a/llvm_extra/create_llvm_extra.sh b/llvm_extra/create_llvm_extra.sh index 6f34a0b2..b58e0508 100755 --- a/llvm_extra/create_llvm_extra.sh +++ b/llvm_extra/create_llvm_extra.sh @@ -82,7 +82,7 @@ function create_llvm_extra_ebuild() { set -e -# Sanity checks. +# Confidence checks. check_cmd "${@}" # Create llvm-extra ebuild. create_llvm_extra_ebuild "${@}" diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py index 6941f3d6..5e878816 100755 --- a/llvm_tools/nightly_revert_checker.py +++ b/llvm_tools/nightly_revert_checker.py @@ -33,7 +33,7 @@ State = t.Any def _find_interesting_android_shas(android_llvm_toolchain_dir: str - ) -> t.List[t.Tuple[str, str]]: + ) -> t.List[t.Tuple[str, str]]: llvm_project = os.path.join(android_llvm_toolchain_dir, 'toolchain/llvm-project') @@ -44,7 +44,7 @@ def _find_interesting_android_shas(android_llvm_toolchain_dir: str encoding='utf-8', ).strip() - main_legacy = get_llvm_merge_base('aosp/master-legacy') + main_legacy = get_llvm_merge_base('aosp/master-legacy') # nocheck testing_upstream = get_llvm_merge_base('aosp/testing-upstream') result = [('main-legacy', main_legacy)] @@ -55,8 +55,7 @@ def _find_interesting_android_shas(android_llvm_toolchain_dir: str def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper - ) -> t.List[t.Tuple[str, str]]: - + ) -> t.List[t.Tuple[str, str]]: def parse_ebuild_assignment(line: str) -> str: no_comments = line.split('#')[0] no_assign = no_comments.split('=', 1)[1].strip() @@ -84,12 +83,11 @@ def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper def _find_interesting_chromeos_shas(chromeos_base: str - ) -> t.List[t.Tuple[str, str]]: + ) -> t.List[t.Tuple[str, str]]: llvm_dir = os.path.join(chromeos_base, 'src/third_party/chromiumos-overlay/sys-devel/llvm') candidate_ebuilds = [ - os.path.join(llvm_dir, x) - for x in os.listdir(llvm_dir) + os.path.join(llvm_dir, x) for x in os.listdir(llvm_dir) if '_pre' in x and not os.path.islink(os.path.join(llvm_dir, x)) ] @@ -197,8 +195,10 @@ def find_shas(llvm_dir: str, interesting_shas: t.List[t.Tuple[str, str]], state: State, new_state: State): for friendly_name, sha in interesting_shas: logging.info('Finding reverts across %s (%s)', friendly_name, sha) - all_reverts = revert_checker.find_reverts( - llvm_dir, sha, root='origin/' + git_llvm_rev.MAIN_BRANCH) + all_reverts = revert_checker.find_reverts(llvm_dir, + sha, + root='origin/' + + git_llvm_rev.MAIN_BRANCH) logging.info('Detected the following revert(s) across %s:\n%s', friendly_name, pprint.pformat(all_reverts)) @@ -230,13 +230,12 @@ def do_cherrypick(chroot_path: str, llvm_dir: str, seen.add(friendly_name) for sha, reverted_sha in reverts: try: - get_upstream_patch.get_from_upstream( - chroot_path=chroot_path, - create_cl=True, - start_sha=reverted_sha, - patches=[sha], - reviewers=reviewers, - cc=cc) + get_upstream_patch.get_from_upstream(chroot_path=chroot_path, + create_cl=True, + start_sha=reverted_sha, + patches=[sha], + reviewers=reviewers, + cc=cc) except get_upstream_patch.CherrypickError as e: logging.info('%s, skipping...', str(e)) return new_state @@ -245,7 +244,6 @@ def do_cherrypick(chroot_path: str, llvm_dir: str, def do_email(is_dry_run: bool, llvm_dir: str, repository: str, interesting_shas: t.List[t.Tuple[str, str]], state: State, recipients: _EmailRecipients) -> State: - def prettify_sha(sha: str) -> tiny_render.Piece: rev = get_llvm_hash.GetVersionFrom(llvm_dir, sha) @@ -253,8 +251,8 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str, short_sha = sha[:12] return tiny_render.Switch( text=f'r{rev} ({short_sha})', - html=tiny_render.Link( - href='https://reviews.llvm.org/rG' + sha, inner='r' + str(rev)), + html=tiny_render.Link(href='https://reviews.llvm.org/rG' + sha, + inner='r' + str(rev)), ) def get_sha_description(sha: str) -> tiny_render.Piece: @@ -267,11 +265,12 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str, new_state: State = {} for friendly_name, sha, new_reverts in find_shas(llvm_dir, interesting_shas, state, new_state): - email = _generate_revert_email(repository, friendly_name, sha, prettify_sha, - get_sha_description, new_reverts) + email = _generate_revert_email(repository, friendly_name, sha, + prettify_sha, get_sha_description, + new_reverts) if is_dry_run: - logging.info('Would send email:\nSubject: %s\nBody:\n%s\n', email.subject, - tiny_render.render_text_pieces(email.body)) + logging.info('Would send email:\nSubject: %s\nBody:\n%s\n', + email.subject, tiny_render.render_text_pieces(email.body)) else: logging.info('Sending email with subject %r...', email.subject) _send_revert_email(recipients, email) @@ -281,16 +280,19 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str, def parse_args(argv: t.List[str]) -> t.Any: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( 'action', choices=['cherry-pick', 'email', 'dry-run'], help='Automatically cherry-pick upstream reverts, send an email, or ' 'write to stdout.') - parser.add_argument( - '--state_file', required=True, help='File to store persistent state in.') - parser.add_argument( - '--llvm_dir', required=True, help='Up-to-date LLVM directory to use.') + parser.add_argument('--state_file', + required=True, + help='File to store persistent state in.') + parser.add_argument('--llvm_dir', + required=True, + help='Up-to-date LLVM directory to use.') parser.add_argument('--debug', action='store_true') parser.add_argument( '--reviewers', @@ -309,8 +311,9 @@ def parse_args(argv: t.List[str]) -> t.Any: subparsers.required = True chromeos_subparser = subparsers.add_parser('chromeos') - chromeos_subparser.add_argument( - '--chromeos_dir', required=True, help='Up-to-date CrOS directory to use.') + chromeos_subparser.add_argument('--chromeos_dir', + required=True, + help='Up-to-date CrOS directory to use.') android_subparser = subparsers.add_parser('android') android_subparser.add_argument( @@ -322,7 +325,7 @@ def parse_args(argv: t.List[str]) -> t.Any: def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str] - ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]: + ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]: recipients = reviewers + cc if opts.repository == 'chromeos': chroot_path = opts.chromeos_dir @@ -335,9 +338,9 @@ def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str] chroot_path = opts.android_llvm_toolchain_dir return (chroot_path, _find_interesting_android_shas(chroot_path), - _EmailRecipients( - well_known=[], - direct=['android-llvm-dev@google.com'] + recipients)) + _EmailRecipients(well_known=[], + direct=['android-llvm-dev@google.com'] + + recipients)) else: raise ValueError(f'Unknown repository {opts.repository}') @@ -366,21 +369,19 @@ def main(argv: t.List[str]) -> int: # We want to be as free of obvious side-effects as possible in case something # above breaks. Hence, action as late as possible. if action == 'cherry-pick': - new_state = do_cherrypick( - chroot_path=chroot_path, - llvm_dir=llvm_dir, - interesting_shas=interesting_shas, - state=state, - reviewers=reviewers, - cc=cc) + new_state = do_cherrypick(chroot_path=chroot_path, + llvm_dir=llvm_dir, + interesting_shas=interesting_shas, + state=state, + reviewers=reviewers, + cc=cc) else: - new_state = do_email( - is_dry_run=action == 'dry-run', - llvm_dir=llvm_dir, - repository=repository, - interesting_shas=interesting_shas, - state=state, - recipients=recipients) + new_state = do_email(is_dry_run=action == 'dry-run', + llvm_dir=llvm_dir, + repository=repository, + interesting_shas=interesting_shas, + state=state, + recipients=recipients) _write_state(state_file, new_state) return 0 diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index 66df7c8a..b2c94acd 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -162,7 +162,7 @@ def get_new_gentoo_commits(git_dir: pathlib.Path, 'git', 'log', '--format=%H %s', - f'{most_recent_sha}..origin/master', + f'{most_recent_sha}..origin/master', # nocheck '--', 'dev-lang/rust', ], @@ -225,7 +225,7 @@ def atomically_write_state(state_file: pathlib.Path, state: State) -> None: def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, new_gentoo_commits: List[GitCommit] - ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: + ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: """Creates an email given our new state, if doing so is appropriate.""" subject_pieces = [] body_pieces = [] @@ -269,11 +269,14 @@ def main(argv: List[str]) -> None: logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--state_dir', required=True, help='Directory to store state in.') - parser.add_argument( - '--skip_email', action='store_true', help="Don't send an email.") + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--state_dir', + required=True, + help='Directory to store state in.') + parser.add_argument('--skip_email', + action='store_true', + help="Don't send an email.") parser.add_argument( '--skip_state_update', action='store_true', @@ -334,8 +337,8 @@ def main(argv: List[str]) -> None: logging.info('Skipping state update, as requested') return - newest_sha = ( - new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) + newest_sha = (new_commits[-1].sha + if new_commits else prior_state.last_gentoo_sha) atomically_write_state( state_file, State( diff --git a/setup_chromeos.py b/setup_chromeos.py deleted file mode 100755 index 07c7530d..00000000 --- a/setup_chromeos.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Script to checkout the ChromeOS source. - -This script sets up the ChromeOS source in the given directory, matching a -particular release of ChromeOS. -""" - -from __future__ import print_function - -__author__ = 'raymes@google.com (Raymes Khoury)' - -from datetime import datetime - -import argparse -import os -import pickle -import sys -import tempfile -import time -from cros_utils import command_executer -from cros_utils import logger -from cros_utils import manifest_versions - -GCLIENT_FILE = """solutions = [ - { "name" : "CHROME_DEPS", - "url" : - "svn://svn.chromium.org/chrome-internal/trunk/tools/buildspec/releases/%s", - "custom_deps" : { - "src/third_party/WebKit/LayoutTests": None, - "src-pdf": None, - "src/pdf": None, - }, - "safesync_url": "", - }, -] -""" - -# List of stable versions used for common team image -# Sheriff must update this list when a new common version becomes available -COMMON_VERSIONS = '/home/mobiletc-prebuild/common_images/common_list.txt' - - -def Usage(parser): - parser.print_help() - sys.exit(0) - - -# Get version spec file, either from "paladin" or "buildspec" directory. -def GetVersionSpecFile(version, versions_git): - temp = tempfile.mkdtemp() - commands = ['cd {0}'.format(temp), \ - 'git clone {0} versions'.format(versions_git)] - cmd_executer = command_executer.GetCommandExecuter() - ret = cmd_executer.RunCommands(commands) - err_msg = None - if ret: - err_msg = 'Failed to checkout versions_git - {0}'.format(versions_git) - ret = None - else: - v, m = version.split('.', 1) - paladin_spec = 'paladin/buildspecs/{0}/{1}.xml'.format(v, m) - generic_spec = 'buildspecs/{0}/{1}.xml'.format(v, m) - paladin_path = '{0}/versions/{1}'.format(temp, paladin_spec) - generic_path = '{0}/versions/{1}'.format(temp, generic_spec) - if os.path.exists(paladin_path): - ret = paladin_spec - elif os.path.exists(generic_path): - ret = generic_spec - else: - err_msg = 'No spec found for version {0}'.format(version) - ret = None - # Fall through to clean up. - - commands = ['rm -rf {0}'.format(temp)] - cmd_executer.RunCommands(commands) - if err_msg: - logger.GetLogger().LogFatal(err_msg) - return ret - - -def TimeToCommonVersion(timestamp): - """Convert timestamp to common image version.""" - tdt = datetime.fromtimestamp(float(timestamp)) - with open(COMMON_VERSIONS, 'r', encoding='utf-8') as f: - common_list = pickle.load(f) - for sv in common_list: - sdt = datetime.strptime(sv['date'], '%Y-%m-%d %H:%M:%S.%f') - if tdt >= sdt: - return '%s.%s' % (sv['chrome_major_version'], sv['chromeos_version']) - # should never reach here - logger.GetLogger().LogFatal('No common version for timestamp') - return None - - -def Main(argv): - """Checkout the ChromeOS source.""" - parser = argparse.ArgumentParser() - parser.add_argument( - '--dir', - dest='directory', - help='Target directory for ChromeOS installation.') - parser.add_argument( - '--version', - dest='version', - default='latest_lkgm', - help="""ChromeOS version. Can be: -(1) A release version in the format: 'X.X.X.X' -(2) 'top' for top of trunk -(3) 'latest_lkgm' for the latest lkgm version -(4) 'lkgm' for the lkgm release before timestamp -(5) 'latest_common' for the latest team common stable version -(6) 'common' for the team common stable version before timestamp -Default is 'latest_lkgm'.""") - parser.add_argument( - '--timestamp', - dest='timestamp', - default=None, - help='Timestamps in epoch format. It will check out the' - 'latest LKGM or the latest COMMON version of ChromeOS' - ' before the timestamp. Use in combination with' - ' --version=latest or --version=common. Use ' - '"date -d +%s" to find epoch time') - parser.add_argument( - '--minilayout', - dest='minilayout', - default=False, - action='store_true', - help='Whether to checkout the minilayout (smaller ' - 'checkout).') - parser.add_argument( - '--jobs', '-j', dest='jobs', help='Number of repo sync threads to use.') - parser.add_argument( - '--public', - '-p', - dest='public', - default=False, - action='store_true', - help='Use the public checkout instead of the private ' - 'one.') - - options = parser.parse_args(argv) - - if not options.version: - parser.print_help() - logger.GetLogger().LogFatal('No version specified.') - else: - version = options.version.strip() - - if not options.timestamp: - timestamp = '' - else: - timestamp = options.timestamp.strip() - if version not in ('lkgm', 'common'): - parser.print_help() - logger.GetLogger().LogFatal('timestamp option only applies for ' - 'versions "lkgm" or "common"') - - if not options.directory: - parser.print_help() - logger.GetLogger().LogFatal('No directory specified.') - - directory = options.directory.strip() - - if options.public: - manifest_repo = 'https://chromium.googlesource.com/chromiumos/manifest.git' - versions_repo = ('https://chromium.googlesource.com/' - 'chromiumos/manifest-versions.git') - else: - manifest_repo = ('https://chrome-internal.googlesource.com/chromeos/' - 'manifest-internal.git') - versions_repo = ('https://chrome-internal.googlesource.com/chromeos/' - 'manifest-versions.git') - - if version == 'top': - init = 'repo init -u %s' % manifest_repo - elif version == 'latest_lkgm': - manifests = manifest_versions.ManifestVersions() - version = manifests.TimeToVersionChromeOS(time.mktime(time.gmtime())) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for latest LKGM' % (version, manifest)) - init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % - (versions_repo, version, manifest)) - del manifests - elif version == 'lkgm': - if not timestamp: - parser.print_help() - logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') - manifests = manifest_versions.ManifestVersions() - version = manifests.TimeToVersion(timestamp) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput('found version %s.%s for LKGM at timestamp %s' - % (version, manifest, timestamp)) - init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % - (versions_repo, version, manifest)) - del manifests - elif version == 'latest_common': - version = TimeToCommonVersion(time.mktime(time.gmtime())) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for latest Common image' % (version, manifest)) - init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, - manifest)) - elif version == 'common': - if not timestamp: - parser.print_help() - logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') - version = TimeToCommonVersion(timestamp) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for latest common image ' - 'at timestamp %s' % (version, manifest, timestamp)) - init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, - manifest)) - else: - # user specified a specific version number - version_spec_file = GetVersionSpecFile(version, versions_repo) - if not version_spec_file: - return 1 - init = 'repo init -u %s -m %s' % (versions_repo, version_spec_file) - - if options.minilayout: - init += ' -g minilayout' - - init += ' --repo-url=https://chromium.googlesource.com/external/repo.git' - - # crosbug#31837 - "Sources need to be world-readable to properly - # function inside the chroot" - sync = 'umask 022 && repo sync' - if options.jobs: - sync += ' -j %s' % options.jobs - - commands = ['mkdir -p %s' % directory, 'cd %s' % directory, init, sync] - cmd_executer = command_executer.GetCommandExecuter() - ret = cmd_executer.RunCommands(commands) - if ret: - return ret - - return cmd_executer.RunCommand( - 'git ls-remote ' - 'https://chrome-internal.googlesource.com/chrome/src-internal.git ' - 'refs/HEAD > /dev/null') - - -if __name__ == '__main__': - retval = Main(sys.argv[1:]) - sys.exit(retval) diff --git a/unblocked_terms.txt b/unblocked_terms.txt deleted file mode 100644 index c961e667..00000000 --- a/unblocked_terms.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Code uses this in many cases to refer to git branches. -master - -# A few paths in AOSP contain this term. -native -- cgit v1.2.3 From 58bc0488c6bd13be3aa00f5a056f19a7ab18980b Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Thu, 28 Oct 2021 20:33:57 +0000 Subject: crosperf: Update default_remotes to new DUTs DUTs have been moved to different rows and racks. This updates the default_remotes accordingly. These devices are chosen by checking the swarming dimension setting "label-pool=toolchain". Additionally, this removes Lulu machines as they are no longer in the toolchain pool. BUG=b:178744715 TEST=Presubmit hooks Change-Id: Ifefb6cb6d2dae58d3c27b55f3a39bccf2fd27eb4 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3252241 Commit-Queue: Jordan R Abrahams Tested-by: Jordan R Abrahams Reviewed-by: Manoj Gupta --- PRESUBMIT.cfg | 4 ++++ crosperf/default_remotes | 15 ++++++------ crosperf/experiment_factory_unittest.py | 41 ++++++++++++++++++--------------- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/PRESUBMIT.cfg b/PRESUBMIT.cfg index d69d375b..5fccf3ac 100644 --- a/PRESUBMIT.cfg +++ b/PRESUBMIT.cfg @@ -1,2 +1,6 @@ [Hook Scripts] toolchain_utils_presubmits = ./toolchain_utils_githooks/check-presubmit ${PRESUBMIT_FILES} + +[Hook Overrides Options] +cros_license_check: --exclude_regex=\b(default_remotes)$ +long_line_check: --exclude_regex=\b(default_remotes)$ diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 45110752..1cc628f9 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,9 +1,8 @@ -bob : chromeos2-row10-rack9-host3.cros chromeos6-row3-rack13-host15.cros -chell : chromeos2-row9-rack9-host1.cros chromeos2-row9-rack9-host3.cros -coral : chromeos2-row9-rack9-host9.cros chromeos2-row9-rack9-host11.cros chromeos2-row9-rack9-host13.cros +bob : chromeos6-row4-rack13-host6.cros chromeos6-row3-rack13-host15.cros +chell : chromeos2-row1-rack10-host2.cros chromeos2-row1-rack10-host4.cros +coral : chromeos6-row5-rack6-host1.cros chromeos6-row5-rack6-host3.cros chromeos6-row5-rack6-host5.cros elm : chromeos6-row14-rack15-host21.cros -kefka : chromeos6-row6-rack22-host2.cros chromeos6-row6-rack22-host3.cros -lulu : chromeos2-row9-rack9-host5.cros chromeos2-row9-rack9-host7.cros -nautilus : chromeos2-row10-rack9-host9.cros chromeos2-row10-rack9-host11.cros -snappy : chromeos2-row10-rack9-host5.cros chromeos2-row10-rack9-host7.cros -veyron_tiger : chromeos2-row9-rack9-host17.cros +kefka : chromeos6-row6-rack22-host2.cros chromeos6-row6-rack22-host3.cros chromeos6-row11-rack22-host7.cros +nautilus : chromeos6-row5-rack10-host1.cros chromeos6-row5-rack10-host3.cros +snappy : chromeos6-row3-rack20-host1.cros chromeos6-row3-rack20-host3.cros +veyron_tiger : chromeos6-row3-rack7-host1.cros diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 78cf780c..9637c108 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -79,14 +79,14 @@ EXPERIMENT_FILE_2 = """ class ExperimentFactoryTest(unittest.TestCase): """Class for running experiment factory unittests.""" - def setUp(self): self.append_benchmark_call_args = [] def testLoadExperimentFile1(self): experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1)) - exp = ExperimentFactory().GetExperiment( - experiment_file, working_directory='', log_dir='') + exp = ExperimentFactory().GetExperiment(experiment_file, + working_directory='', + log_dir='') self.assertEqual(exp.remote, ['chromeos-alex3']) self.assertEqual(len(exp.benchmarks), 2) @@ -104,8 +104,9 @@ class ExperimentFactoryTest(unittest.TestCase): def testLoadExperimentFile2CWP(self): experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_2)) - exp = ExperimentFactory().GetExperiment( - experiment_file, working_directory='', log_dir='') + exp = ExperimentFactory().GetExperiment(experiment_file, + working_directory='', + log_dir='') self.assertEqual(exp.cwp_dso, 'kallsyms') self.assertEqual(len(exp.benchmarks), 2) self.assertEqual(exp.benchmarks[0].weight, 0.8) @@ -240,11 +241,12 @@ class ExperimentFactoryTest(unittest.TestCase): ef = ExperimentFactory() bench_list = [] - ef.AppendBenchmarkSet(bench_list, experiment_factory.telemetry_perfv2_tests, - '', 1, False, '', 'telemetry_Crosperf', False, 0, - False, '', 0) - self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_perfv2_tests)) + ef.AppendBenchmarkSet(bench_list, + experiment_factory.telemetry_perfv2_tests, '', 1, + False, '', 'telemetry_Crosperf', False, 0, False, '', + 0) + self.assertEqual(len(bench_list), + len(experiment_factory.telemetry_perfv2_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] @@ -252,17 +254,17 @@ class ExperimentFactoryTest(unittest.TestCase): experiment_factory.telemetry_pagecycler_tests, '', 1, False, '', 'telemetry_Crosperf', False, 0, False, '', 0) - self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) + self.assertEqual(len(bench_list), + len(experiment_factory.telemetry_pagecycler_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] ef.AppendBenchmarkSet(bench_list, - experiment_factory.telemetry_toolchain_perf_tests, '', - 1, False, '', 'telemetry_Crosperf', False, 0, False, - '', 0) - self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests)) + experiment_factory.telemetry_toolchain_perf_tests, + '', 1, False, '', 'telemetry_Crosperf', False, 0, + False, '', 0) + self.assertEqual(len(bench_list), + len(experiment_factory.telemetry_toolchain_perf_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) @mock.patch.object(socket, 'gethostname') @@ -370,7 +372,8 @@ class ExperimentFactoryTest(unittest.TestCase): global_settings.SetField('same_machine', 'true') global_settings.SetField('same_specs', 'true') - self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '') + self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', + '') label_settings.SetField('remote', '') global_settings.SetField('remote', '123.45.67.89') exp = ef.GetExperiment(mock_experiment_file, '', '') @@ -399,7 +402,7 @@ class ExperimentFactoryTest(unittest.TestCase): def test_get_default_remotes(self): board_list = [ - 'bob', 'chell', 'coral', 'elm', 'kefka', 'lulu', 'nautilus', 'snappy', + 'bob', 'chell', 'coral', 'elm', 'kefka', 'nautilus', 'snappy', 'veyron_tiger' ] -- cgit v1.2.3 From c965dab625dd16e8983a2865ed2d6dca7291d4ff Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Wed, 3 Nov 2021 13:28:55 -0700 Subject: afdo_metadata: Publish the new kernel profiles Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 Update chromeos-kernel-5_4 BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator Change-Id: I59e1e016248a80227b6657236b72bc73168bfa6d Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3260685 Tested-by: Denis Nikitin Auto-Submit: Denis Nikitin Reviewed-by: George Burgess Commit-Queue: George Burgess --- afdo_metadata/kernel_afdo.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 982770f9..b2aae19b 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,14 @@ { "chromeos-kernel-4_4": { - "name": "R97-14263.0-1633944739" + "name": "R97-14309.0-1635759471" }, "chromeos-kernel-4_14": { - "name": "R97-14263.0-1633944973" + "name": "R97-14309.0-1635759628" }, "chromeos-kernel-4_19": { - "name": "R97-14263.0-1633944768" + "name": "R97-14309.0-1635759164" }, "chromeos-kernel-5_4": { - "name": "R97-14263.0-1633944667" + "name": "R97-14309.0-1635759310" } } -- cgit v1.2.3 From 28d6cbe1b03d7276fca88f1ab6d90e7914092d65 Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Thu, 4 Nov 2021 00:13:50 +0000 Subject: crosperf: Replace non-inclusive language Missed a single instance of this, as it's not a violation in go/pi-terms, but is a violation in go/coil. BUG=b:204057159 TEST=Presubmit tests Change-Id: I4bc4997cb42c82c66e4d56281bb16e4452072e01 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3260693 Tested-by: Jordan R Abrahams Auto-Submit: Jordan R Abrahams Commit-Queue: George Burgess Reviewed-by: George Burgess --- crosperf/results_cache_unittest.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index 94b104c2..d6953eed 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -180,7 +180,7 @@ PERF_DATA_HEADER = """ # total memory : 5911496 kB # cmdline : /usr/bin/perf record -e instructions -p {pid} # event : name = instructions, , id = ( 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193 ), type = 8, size = 112 -# event : name = dummy:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9 +# event : name = placeholder:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9 # CPU_TOPOLOGY info available, use -I to display # pmu mappings: software = 1, uprobe = 6, cs_etm = 8, breakpoint = 5, tracepoint = 2, armv8_pmuv3 = 7 # contains AUX area data (e.g. instruction trace) @@ -442,7 +442,6 @@ HISTOGRAMSET = (""" class MockResult(Result): """Mock result class.""" - def __init__(self, mylogger, label, logging_level, machine): super(MockResult, self).__init__(mylogger, label, logging_level, machine) @@ -458,7 +457,6 @@ class MockResult(Result): class ResultTest(unittest.TestCase): """Result test class.""" - def __init__(self, *args, **kwargs): super(ResultTest, self).__init__(*args, **kwargs) self.callFakeProcessResults = False @@ -1867,7 +1865,6 @@ page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-t class TelemetryResultTest(unittest.TestCase): """Telemetry result test.""" - def __init__(self, *args, **kwargs): super(TelemetryResultTest, self).__init__(*args, **kwargs) self.callFakeProcessResults = False @@ -1912,7 +1909,6 @@ class TelemetryResultTest(unittest.TestCase): class ResultsCacheTest(unittest.TestCase): """Resultcache test class.""" - def __init__(self, *args, **kwargs): super(ResultsCacheTest, self).__init__(*args, **kwargs) self.fakeCacheReturnResult = None -- cgit v1.2.3 From 58a779e703fff9d349b33de2bd2bfe25c6423729 Mon Sep 17 00:00:00 2001 From: Christopher Di Bella Date: Mon, 1 Nov 2021 17:36:00 +0000 Subject: llvm_tools: adds a way to skip dependencies when pulling from Phab Phabricator has a way to list dependencies, which Arcanist will pull in succession to make sure the patch is correct. This is fine until some of the patches land in main: Arcanist then gets confused and doesn't always correctly apply the patches. This patch makes it possible to skip dependencies by adding a new flag to our upstream patch getter. The current design is only going to permit `--skip_dependencies` when there's exactly one `--differential` patch, since `--skip_dependencies` is a global option. If we're finding this to be a common nuisance, then we can look at redesigning the feature to allow lists of differentials with skipped dependencies. ``` ./get_upstream_patch.py --differential D12345 --skip_dependencies ./get_upstream_patch.py --cl 12345 --differential D45678 --skip_dependencies ./get_upstream_patch.py --cl 12345 --skip-dependencies ./get_upstream_patch.py --differential D12345 \ --differential D45678 \ --skip_dependencies ``` BUG=b:204779256 TEST=Tested locally Change-Id: I89032cba82c17c9f844bfac3cba330d173826ffd Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3255665 Commit-Queue: Christopher Di Bella Tested-by: Christopher Di Bella Auto-Submit: Christopher Di Bella Reviewed-by: Manoj Gupta Reviewed-by: George Burgess --- llvm_tools/get_upstream_patch.py | 89 +++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index 77783d41..cc5505db 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -31,8 +31,8 @@ class CherrypickError(ValueError): def add_patch(patches_json_path: str, patches_dir: str, relative_patches_dir: str, start_version: git_llvm_rev.Rev, - llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str, - package: str): + llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, + str], sha: str, package: str): """Gets the start and end intervals in 'json_file'. Args: @@ -85,7 +85,9 @@ def add_patch(patches_json_path: str, patches_dir: str, subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) commit_subject = subprocess.check_output( - ['git', 'log', '-n1', '--format=%s', sha], cwd=llvm_dir, encoding='utf-8') + ['git', 'log', '-n1', '--format=%s', sha], + cwd=llvm_dir, + encoding='utf-8') patch_metadata = { 'comment': commit_subject.strip(), @@ -177,8 +179,8 @@ def get_package_names(sha: str, llvm_dir: str) -> list: def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], start_rev: git_llvm_rev.Rev, - rev: t.Union[git_llvm_rev.Rev, str], sha: str, - llvm_dir: str): + rev: t.Union[git_llvm_rev.Rev, + str], sha: str, llvm_dir: str): """Create a patch and add its metadata for each package""" for package, symlink in zip(packages, symlinks): symlink_dir = os.path.dirname(symlink) @@ -212,12 +214,11 @@ def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str: return start_sha -def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], - start_rev: git_llvm_rev.Rev, - llvm_config: git_llvm_rev.LLVMConfig, - llvm_symlink_dir: str, create_cl: bool, - reviewers: t.Optional[t.List[str]], - cc: t.Optional[t.List[str]]): +def find_patches_and_make_cl( + chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev, + llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, + create_cl: bool, skip_dependencies: bool, + reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]]): if create_cl: branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' git.CreateBranch(llvm_symlink_dir, branch) @@ -231,7 +232,11 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], is_differential = patch.startswith('D') if is_differential: subprocess.check_output( - ['arc', 'patch', '--nobranch', '--revision', patch], + [ + 'arc', 'patch', '--nobranch', + '--skip-dependencies' if skip_dependencies else '--revision', + patch + ], cwd=llvm_config.dir, ) sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') @@ -275,6 +280,7 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], def get_from_upstream(chroot_path: str, create_cl: bool, + skip_dependencies: bool, start_sha: str, patches: t.List[str], reviewers: t.List[str] = None, @@ -299,15 +305,16 @@ def get_from_upstream(chroot_path: str, remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) - find_patches_and_make_cl( - chroot_path=chroot_path, - patches=patches, - start_rev=git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha), - llvm_config=llvm_config, - llvm_symlink_dir=llvm_symlink_dir, - create_cl=create_cl, - reviewers=reviewers, - cc=cc) + find_patches_and_make_cl(chroot_path=chroot_path, + patches=patches, + start_rev=git_llvm_rev.translate_sha_to_rev( + llvm_config, start_sha), + llvm_config=llvm_config, + llvm_symlink_dir=llvm_symlink_dir, + create_cl=create_cl, + skip_dependencies=skip_dependencies, + reviewers=reviewers, + cc=cc) logging.info('Complete.') @@ -319,40 +326,46 @@ def main(): ) parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--chroot_path', - default=os.path.join(os.path.expanduser('~'), 'chromiumos'), - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=os.path.join(os.path.expanduser('~'), + 'chromiumos'), + help='the path to the chroot (default: %(default)s)') parser.add_argument( '--start_sha', default='llvm-next', help='LLVM SHA that the patch should start applying at. You can specify ' '"llvm" or "llvm-next", as well. Defaults to %(default)s.') - parser.add_argument( - '--sha', - action='append', - default=[], - help='The LLVM git SHA to cherry-pick.') + parser.add_argument('--sha', + action='append', + default=[], + help='The LLVM git SHA to cherry-pick.') parser.add_argument( '--differential', action='append', default=[], help='The LLVM differential revision to apply. Example: D1234') + parser.add_argument('--create_cl', + action='store_true', + help='Automatically create a CL if specified') parser.add_argument( - '--create_cl', - default=False, + '--skip_dependencies', action='store_true', - help='Automatically create a CL if specified') + help="Skips a LLVM differential revision's dependencies. Only valid " + 'when --differential appears exactly once.') args = parser.parse_args() if not (args.sha or args.differential): parser.error('--sha or --differential required') - get_from_upstream( - chroot_path=args.chroot_path, - create_cl=args.create_cl, - start_sha=args.start_sha, - patches=args.sha + args.differential) + if args.skip_dependencies and len(args.differential) != 1: + parser.error("--skip_dependencies is only valid when there's exactly one " + 'supplied differential') + + get_from_upstream(chroot_path=args.chroot_path, + create_cl=args.create_cl, + skip_dependencies=args.skip_dependencies, + start_sha=args.start_sha, + patches=args.sha + args.differential) if __name__ == '__main__': -- cgit v1.2.3 From 9de6ecbfb6b2a99ea734f353623a663247a11b5c Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Fri, 5 Nov 2021 13:31:19 -0700 Subject: crosperf: Increase timeout for a crosfleet call Current timeout 2 min may not be sufficiant for `crosfleet dut lease`. Many tests fail with the lock error while lease succeeds. Increase the timeout to 8 minutes. Remove backslashes and fix other minor lints to make upload happy. BUG=b:205325997 TEST=run crosperf locally on veyron Change-Id: Ie3b74a0036cce3daeb58379fc5362797b0e5db03 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3265564 Tested-by: Denis Nikitin Commit-Queue: Denis Nikitin Reviewed-by: Manoj Gupta --- lock_machine.py | 123 ++++++++++++++++++++++++++------------------------------ 1 file changed, 56 insertions(+), 67 deletions(-) diff --git a/lock_machine.py b/lock_machine.py index e7020591..03c8c991 100755 --- a/lock_machine.py +++ b/lock_machine.py @@ -125,8 +125,8 @@ class LockManager(object): Returns: A list of names of the toolchain machines in the ChromeOS HW lab. """ - machines_file = os.path.join( - os.path.dirname(__file__), 'crosperf', 'default_remotes') + machines_file = os.path.join(os.path.dirname(__file__), 'crosperf', + 'default_remotes') machine_list = [] with open(machines_file, 'r') as input_file: lines = input_file.readlines() @@ -166,8 +166,8 @@ class LockManager(object): """ if state['locked']: print('%s (%s)\t\t%slocked by %s since %s' % - (m, state['board'], '\t\t' if machine_type == MachineType.LOCAL else - '', state['locked_by'], state['lock_time'])) + (m, state['board'], '\t\t' if machine_type == MachineType.LOCAL + else '', state['locked_by'], state['lock_time'])) else: print('%s (%s)\t\t%sunlocked' % (m, state['board'], @@ -241,8 +241,8 @@ class LockManager(object): """ try: if should_lock_machine: - ret = file_lock_machine.Machine(machine, - self.locks_dir).Lock(True, sys.argv[0]) + ret = file_lock_machine.Machine(machine, self.locks_dir).Lock( + True, sys.argv[0]) else: ret = file_lock_machine.Machine(machine, self.locks_dir).Unlock(True) except Exception: @@ -325,14 +325,14 @@ class LockManager(object): # TODO(zhizhouy): Crosfleet doesn't support host info such as locked_by. # Need to update this when crosfleet supports it. - if (state['locked'] and state['locked_by'] and - state['locked_by'] != self.user): + if (state['locked'] and state['locked_by'] + and state['locked_by'] != self.user): raise DontOwnLock('Attempt to unlock machine (%s) locked by someone ' 'else (%s).' % (k, state['locked_by'])) elif cmd == 'lock': if state['locked']: - self.logger.LogWarning('Attempt to lock already locked machine (%s)' % - k) + self.logger.LogWarning( + 'Attempt to lock already locked machine (%s)' % k) self._InternalRemoveMachine(k) def GetMachineStates(self, cmd=''): @@ -381,12 +381,10 @@ class LockManager(object): swarming = os.path.join(self.chromeos_root, self.SWARMING) # TODO(zhizhouy): Swarming script doesn't support python3 so explicitly # launch it with python2 until migrated. - cmd = (('python2 %s ' \ - 'query --swarming https://chromeos-swarming.appspot.com ' \ - "%s 'bots/list?is_dead=FALSE&dimensions=dut_name:%s'") % \ - (swarming, - credential, - machine.rstrip('.cros'))) + cmd = (('python2 %s ' + 'query --swarming https://chromeos-swarming.appspot.com ' + "%s 'bots/list?is_dead=FALSE&dimensions=dut_name:%s'") % + (swarming, credential, machine.rstrip('.cros'))) exit_code, stdout, stderr = self.ce.RunCommandWOutput(cmd) if exit_code: raise ValueError('Querying bots failed (2); stdout: %r; stderr: %r' % @@ -411,15 +409,12 @@ class LockManager(object): credential = '' if os.path.exists(self.CROSFLEET_CREDENTIAL): credential = '-service-account-json %s' % self.CROSFLEET_CREDENTIAL - cmd = (('%s dut lease -minutes %s %s %s %s') % \ - (self.CROSFLEET_PATH, - self.LEASE_MINS, - credential, - '-host' if '.cros' in machine else '-board', - machine.rstrip('.cros'))) - # Wait 120 seconds for server to start the lease task, if not started, + cmd = (('%s dut lease -minutes %s %s %s %s') % + (self.CROSFLEET_PATH, self.LEASE_MINS, credential, '-host' + if '.cros' in machine else '-board', machine.rstrip('.cros'))) + # Wait 8 minutes for server to start the lease task, if not started, # we will treat it as unavailable. - check_interval_time = 120 + check_interval_time = 480 retval = self.ce.RunCommand(cmd, command_timeout=check_interval_time) return retval == self.SUCCESS @@ -432,10 +427,8 @@ class LockManager(object): credential = '' if os.path.exists(self.CROSFLEET_CREDENTIAL): credential = '-service-account-json %s' % self.CROSFLEET_CREDENTIAL - cmd = (('%s dut abandon %s %s') % \ - (self.CROSFLEET_PATH, - credential, - machine.rstrip('.cros'))) + cmd = (('%s dut abandon %s %s') % + (self.CROSFLEET_PATH, credential, machine.rstrip('.cros'))) retval = self.ce.RunCommand(cmd) return retval == self.SUCCESS @@ -451,44 +444,39 @@ def Main(argv): """ parser = argparse.ArgumentParser() - parser.add_argument( - '--list', - dest='cmd', - action='store_const', - const='status', - help='List current status of all known machines.') - parser.add_argument( - '--lock', - dest='cmd', - action='store_const', - const='lock', - help='Lock given machine(s).') - parser.add_argument( - '--unlock', - dest='cmd', - action='store_const', - const='unlock', - help='Unlock given machine(s).') - parser.add_argument( - '--status', - dest='cmd', - action='store_const', - const='status', - help='List current status of given machine(s).') - parser.add_argument( - '--remote', dest='remote', help='machines on which to operate') - parser.add_argument( - '--chromeos_root', - dest='chromeos_root', - required=True, - help='ChromeOS root to use for autotest scripts.') - parser.add_argument( - '--force', - dest='force', - action='store_true', - default=False, - help='Force lock/unlock of machines, even if not' - ' current lock owner.') + parser.add_argument('--list', + dest='cmd', + action='store_const', + const='status', + help='List current status of all known machines.') + parser.add_argument('--lock', + dest='cmd', + action='store_const', + const='lock', + help='Lock given machine(s).') + parser.add_argument('--unlock', + dest='cmd', + action='store_const', + const='unlock', + help='Unlock given machine(s).') + parser.add_argument('--status', + dest='cmd', + action='store_const', + const='status', + help='List current status of given machine(s).') + parser.add_argument('--remote', + dest='remote', + help='machines on which to operate') + parser.add_argument('--chromeos_root', + dest='chromeos_root', + required=True, + help='ChromeOS root to use for autotest scripts.') + parser.add_argument('--force', + dest='force', + action='store_true', + default=False, + help='Force lock/unlock of machines, even if not' + ' current lock owner.') options = parser.parse_args(argv) @@ -506,7 +494,8 @@ def Main(argv): if options.remote: machine_list = options.remote.split() - lock_manager = LockManager(machine_list, options.force, options.chromeos_root) + lock_manager = LockManager(machine_list, options.force, + options.chromeos_root) machine_states = lock_manager.GetMachineStates(cmd=options.cmd) cmd = options.cmd -- cgit v1.2.3 From 7afc35229485f3a72173a9f11bbb5be56df49dc4 Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Mon, 15 Nov 2021 19:25:28 +0000 Subject: llvm_tools: Make skip_dependencies optional This CL makes new skip_denedencies arg in get_upstream patch default to False so that the new behavior introduced in CL:3255665 is disabled by default. This is intended to fix a failure in the nightly revert checker that was preventing cherry_picks. BUG=b:206461485 TEST=Local test of get_upstream_patch.py, and nightly_revert_checker_test.py Change-Id: Ia7586b69f9aefb5ca820a60eed593d45a4b9a5bb Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3282951 Reviewed-by: Christopher Di Bella Commit-Queue: Christopher Di Bella Tested-by: Christopher Di Bella Auto-Submit: Ryan Beltran --- llvm_tools/get_upstream_patch.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index cc5505db..f72891e0 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -280,9 +280,9 @@ def find_patches_and_make_cl( def get_from_upstream(chroot_path: str, create_cl: bool, - skip_dependencies: bool, start_sha: str, patches: t.List[str], + skip_dependencies: bool = False, reviewers: t.List[str] = None, cc: t.List[str] = None): llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( @@ -361,11 +361,13 @@ def main(): parser.error("--skip_dependencies is only valid when there's exactly one " 'supplied differential') - get_from_upstream(chroot_path=args.chroot_path, - create_cl=args.create_cl, - skip_dependencies=args.skip_dependencies, - start_sha=args.start_sha, - patches=args.sha + args.differential) + get_from_upstream( + chroot_path=args.chroot_path, + create_cl=args.create_cl, + start_sha=args.start_sha, + patches=args.sha + args.differential, + skip_dependencies=args.skip_dependencies, + ) if __name__ == '__main__': -- cgit v1.2.3 From 50d427051d156305f61e710d369fdef7171ad931 Mon Sep 17 00:00:00 2001 From: Manoj Gupta Date: Thu, 2 Dec 2021 10:56:01 -0800 Subject: afdo_metadata: Publish the new kernel profiles Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 Update chromeos-kernel-5_4 BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator Change-Id: Ib50ce95a43740d633f4db9601f09d76126bf9abc Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3313556 Commit-Queue: Manoj Gupta Commit-Queue: Denis Nikitin Tested-by: Manoj Gupta Auto-Submit: Manoj Gupta Reviewed-by: Denis Nikitin --- afdo_metadata/kernel_afdo.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index b2aae19b..98a00881 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,14 @@ { "chromeos-kernel-4_4": { - "name": "R97-14309.0-1635759471" + "name": "R98-14324.19-1638181896" }, "chromeos-kernel-4_14": { - "name": "R97-14309.0-1635759628" + "name": "R98-14350.0-1638182285" }, "chromeos-kernel-4_19": { - "name": "R97-14309.0-1635759164" + "name": "R98-14352.0-1638182035" }, "chromeos-kernel-5_4": { - "name": "R97-14309.0-1635759310" + "name": "R98-14352.0-1638181967" } } -- cgit v1.2.3 From d02a62da66a5ca7d7d849aca8289ca783a68a7c8 Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Mon, 6 Dec 2021 23:06:12 +0000 Subject: compiler_wrapper: add missing unused-but-set-var This CL adds a Wno-unused-but-set-var flag that was accidentally missing in a clangPostFlags field. THis is already present in the llvm repo, must have been fixed in there without mirroring it. BUG=b:195433889 TEST=None Change-Id: I23fcc91aee7b316cd70652f0119177d5ebdf7bbd Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3319338 Reviewed-by: Manoj Gupta Commit-Queue: Ryan Beltran Tested-by: Ryan Beltran --- compiler_wrapper/config.go | 1 + .../testdata/cros_clang_host_golden/bisect.json | 9 ++++-- .../clang_ftrapv_maincc_target_specific.json | 27 ++++++++++------ .../cros_clang_host_golden/clang_host_wrapper.json | 3 +- .../clang_maincc_target_specific.json | 27 ++++++++++------ .../cros_clang_host_golden/clang_path.json | 36 ++++++++++++++-------- .../clang_sanitizer_args.json | 24 ++++++++++----- .../clang_specific_args.json | 12 +++++--- .../testdata/cros_clang_host_golden/clangtidy.json | 24 ++++++++++----- .../force_disable_werror.json | 11 +++++-- 10 files changed, 117 insertions(+), 57 deletions(-) diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go index 409456f8..6c28287c 100644 --- a/compiler_wrapper/config.go +++ b/compiler_wrapper/config.go @@ -245,6 +245,7 @@ var crosHostConfig = &config{ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", }, newWarningsDir: "/tmp/fatal_clang_warnings", triciumNitsDir: "/tmp/linting_output/clang-tidy", diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json index fae0f9f5..e237c7c7 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json @@ -43,7 +43,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" @@ -97,7 +98,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" @@ -154,7 +156,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json index 947f1a4d..07fccc6d 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json @@ -34,7 +34,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -75,7 +76,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -116,7 +118,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -157,7 +160,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -198,7 +202,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -239,7 +244,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -280,7 +286,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -321,7 +328,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -362,7 +370,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json index 9267f5b4..a221605e 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json @@ -33,7 +33,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json index c7a4a6a7..2130d528 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json @@ -33,7 +33,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -73,7 +74,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -113,7 +115,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -153,7 +156,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -193,7 +197,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -233,7 +238,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -273,7 +279,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -313,7 +320,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -353,7 +361,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json index 96b3fe50..43ae728a 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json @@ -33,7 +33,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -76,7 +77,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", @@ -119,7 +121,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -162,7 +165,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -214,7 +218,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -269,7 +274,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -324,7 +330,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", @@ -367,7 +374,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -407,7 +415,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -447,7 +456,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -487,7 +497,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -530,7 +541,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json index 14e45b27..cc64bad9 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json @@ -36,7 +36,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -79,7 +80,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -122,7 +124,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -165,7 +168,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -208,7 +212,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -253,7 +258,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -295,7 +301,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -337,7 +344,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json index 01a50a96..7c4afd32 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json @@ -51,7 +51,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -93,7 +94,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -135,7 +137,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -177,7 +180,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json index 2a563b89..f678ba64 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json @@ -49,7 +49,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } }, @@ -76,7 +77,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -133,7 +135,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } }, @@ -160,7 +163,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -219,7 +223,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", @@ -249,7 +254,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -309,7 +315,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } }, @@ -336,7 +343,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json index 139e351a..f5a77714 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json @@ -36,7 +36,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -79,7 +80,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stderr": "-Werror originalerror", @@ -109,6 +111,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-Wno-error", "-Wno-error=poison-system-directories" ] @@ -155,7 +158,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stderr": "-Werror originalerror", @@ -185,6 +189,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-Wno-error", "-Wno-error=poison-system-directories" ] -- cgit v1.2.3 From 86fe58c1efd0db1d0d95b3cb0d6c34d105b39126 Mon Sep 17 00:00:00 2001 From: Christopher Di Bella Date: Tue, 7 Dec 2021 19:54:18 +0000 Subject: compiler_wrapper: adds ${sysroot}/usr/lib to linker path This change has been in chromiumos-overlay for a while now, but it apparently wasn't ever committed to toolchain-utils. BUG=b:209649372 TEST=This has been in prod for ages. Change-Id: Ied0b519f2f30ef275e921f41e6dbdee9c9442d03 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3321723 Tested-by: Christopher Di Bella Auto-Submit: Christopher Di Bella Reviewed-by: Manoj Gupta Commit-Queue: Manoj Gupta --- compiler_wrapper/sysroot_flag.go | 13 +++++++++++++ .../testdata/cros_hardened_golden/bisect.json | 3 +++ .../clang_ftrapv_maincc_target_specific.json | 9 +++++++++ .../clang_maincc_target_specific.json | 9 +++++++++ .../testdata/cros_hardened_golden/clang_path.json | 12 ++++++++++++ .../cros_hardened_golden/clang_sanitizer_args.json | 8 ++++++++ .../cros_hardened_golden/clang_specific_args.json | 4 ++++ .../clang_sysroot_wrapper_common.json | 7 +++++++ .../testdata/cros_hardened_golden/clangtidy.json | 8 ++++++++ .../cros_hardened_golden/force_disable_werror.json | 5 +++++ .../cros_hardened_golden/gcc_clang_syntax.json | 7 +++++++ .../gcc_maincc_target_specific.json | 21 +++++++++++++++------ .../testdata/cros_hardened_golden/gcc_path.json | 6 ++++++ .../cros_hardened_golden/gcc_sanitizer_args.json | 8 ++++++++ .../cros_hardened_golden/gcc_specific_args.json | 3 +++ .../gcc_sysroot_wrapper_common.json | 9 ++++++++- .../cros_hardened_llvmnext_golden/bisect.json | 3 +++ .../cros_hardened_llvmnext_golden/clang_path.json | 12 ++++++++++++ .../cros_hardened_llvmnext_golden/clangtidy.json | 8 ++++++++ .../force_disable_werror.json | 5 +++++ .../gcc_clang_syntax.json | 7 +++++++ .../cros_hardened_llvmnext_golden/gcc_path.json | 6 ++++++ .../cros_hardened_noccache_golden/bisect.json | 3 +++ .../cros_hardened_noccache_golden/clang_path.json | 12 ++++++++++++ .../cros_hardened_noccache_golden/clangtidy.json | 8 ++++++++ .../force_disable_werror.json | 5 +++++ .../gcc_clang_syntax.json | 7 +++++++ .../cros_hardened_noccache_golden/gcc_path.json | 6 ++++++ .../testdata/cros_nonhardened_golden/bisect.json | 3 +++ .../clang_ftrapv_maincc_target_specific.json | 9 +++++++++ .../clang_maincc_target_specific.json | 9 +++++++++ .../cros_nonhardened_golden/clang_path.json | 12 ++++++++++++ .../clang_sanitizer_args.json | 8 ++++++++ .../clang_specific_args.json | 4 ++++ .../clang_sysroot_wrapper_common.json | 7 +++++++ .../testdata/cros_nonhardened_golden/clangtidy.json | 8 ++++++++ .../force_disable_werror.json | 5 +++++ .../cros_nonhardened_golden/gcc_clang_syntax.json | 7 +++++++ .../gcc_maincc_target_specific.json | 21 +++++++++++++++------ .../testdata/cros_nonhardened_golden/gcc_path.json | 6 ++++++ .../cros_nonhardened_golden/gcc_sanitizer_args.json | 8 ++++++++ .../cros_nonhardened_golden/gcc_specific_args.json | 3 +++ .../gcc_sysroot_wrapper_common.json | 9 ++++++++- 43 files changed, 319 insertions(+), 14 deletions(-) diff --git a/compiler_wrapper/sysroot_flag.go b/compiler_wrapper/sysroot_flag.go index 501fb20e..e0583b22 100644 --- a/compiler_wrapper/sysroot_flag.go +++ b/compiler_wrapper/sysroot_flag.go @@ -11,9 +11,14 @@ import ( func processSysrootFlag(builder *commandBuilder) { fromUser := false + userSysroot := "" for _, arg := range builder.args { if arg.fromUser && strings.HasPrefix(arg.value, "--sysroot=") { fromUser = true + sysrootArg := strings.Split(arg.value, "=") + if len(sysrootArg) == 2 { + userSysroot = sysrootArg[1] + } break } } @@ -27,5 +32,13 @@ func processSysrootFlag(builder *commandBuilder) { } if !fromUser { builder.addPreUserArgs("--sysroot=" + sysroot) + } else { + sysroot = userSysroot } + + libdir := "-L" + sysroot + "/usr/lib" + if strings.Contains(builder.target.target, "64") { + libdir += "64" + } + builder.addPostUserArgs(libdir) } diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index 995ce860..05aea31f 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -50,6 +50,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -122,6 +123,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -197,6 +199,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json index 86809109..2b04d0c9 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json @@ -42,6 +42,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -104,6 +105,7 @@ "--prefix=../../bin/x86_64-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -166,6 +168,7 @@ "--prefix=../../bin/x86_64-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -228,6 +231,7 @@ "--prefix=../../bin/armv7m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -289,6 +293,7 @@ "--prefix=../../bin/armv7m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -350,6 +355,7 @@ "--prefix=../../bin/armv7m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -411,6 +417,7 @@ "--prefix=../../bin/armv8m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -472,6 +479,7 @@ "--prefix=../../bin/armv8m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -533,6 +541,7 @@ "--prefix=../../bin/armv8m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json index 2aae9cbb..ea834174 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json @@ -40,6 +40,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -100,6 +101,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -160,6 +162,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -220,6 +223,7 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-linux-gnu-", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -279,6 +283,7 @@ "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -338,6 +343,7 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-win-gnu-", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -397,6 +403,7 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-linux-gnu-", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -456,6 +463,7 @@ "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -515,6 +523,7 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-win-gnu-", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json index 3e396ab9..52d4184a 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json @@ -40,6 +40,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -103,6 +104,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -166,6 +168,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -229,6 +232,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -301,6 +305,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -376,6 +381,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -446,6 +452,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -509,6 +516,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -569,6 +577,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -629,6 +638,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -689,6 +699,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -752,6 +763,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json index 38a34c02..948c19b2 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json @@ -42,6 +42,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -104,6 +105,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -166,6 +168,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -228,6 +231,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -290,6 +294,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -354,6 +359,7 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -415,6 +421,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -477,6 +484,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json index 1098b059..d10e6cd0 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json @@ -58,6 +58,7 @@ "-Woverride-init", "-Wunsafe-loop-optimizations", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -120,6 +121,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-#warnings", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -182,6 +184,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-error=uninitialized", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -244,6 +247,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-someflag", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json index 6ea77a71..42209185 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -77,6 +78,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -140,6 +142,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -194,6 +197,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -254,6 +258,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -314,6 +319,7 @@ "--prefix=../../bin/armv7a-cros-linux-gnueabihf-", "-D__KERNEL__", "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -374,6 +380,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json index 0d4ee88f..ea1363e7 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json @@ -55,6 +55,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -96,6 +97,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -167,6 +169,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -209,6 +212,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -282,6 +286,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -327,6 +332,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -401,6 +407,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -443,6 +450,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json index fc55b4cd..4df81578 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json @@ -43,6 +43,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -106,6 +107,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -155,6 +157,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -222,6 +225,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -271,6 +275,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json index ede8dd00..e2037e26 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json @@ -40,6 +40,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -71,6 +72,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -126,6 +128,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -157,6 +160,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -207,6 +211,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -270,6 +275,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -301,6 +307,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json index 92fec35e..0cc3d8a9 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -65,6 +66,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -103,6 +105,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -140,7 +143,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -177,7 +181,8 @@ "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -214,7 +219,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -251,7 +257,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -288,7 +295,8 @@ "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -325,7 +333,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json index f44e69dd..5b129d7d 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -68,6 +69,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -109,6 +111,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -147,6 +150,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -185,6 +189,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -226,6 +231,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json index 96215022..a61ead8e 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json @@ -30,6 +30,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -71,6 +72,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -112,6 +114,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -153,6 +156,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -193,6 +197,7 @@ "-static-libgcc", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -235,6 +240,7 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -275,6 +281,7 @@ "-static-libgcc", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -315,6 +322,7 @@ "-static-libgcc", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json index f5c311b4..f7b20001 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json @@ -29,6 +29,7 @@ "-static-libgcc", "-march=goldmont", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -69,6 +70,7 @@ "-static-libgcc", "-march=goldmont-plus", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -109,6 +111,7 @@ "-static-libgcc", "-march=skylake", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json index 9ca8403e..299b46b6 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -64,6 +65,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -105,6 +107,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -138,6 +141,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -176,6 +180,7 @@ "-fno-stack-protector", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -213,7 +218,8 @@ "-mthumb", "-fno-stack-protector", "-D__KERNEL__", - "main.cc" + "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -252,6 +258,7 @@ "-static-libgcc", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index 995ce860..05aea31f 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -50,6 +50,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -122,6 +123,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -197,6 +199,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json index 3e396ab9..52d4184a 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json @@ -40,6 +40,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -103,6 +104,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -166,6 +168,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -229,6 +232,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -301,6 +305,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -376,6 +381,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -446,6 +452,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -509,6 +516,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -569,6 +577,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -629,6 +638,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -689,6 +699,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -752,6 +763,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json index 0d4ee88f..ea1363e7 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json @@ -55,6 +55,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -96,6 +97,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -167,6 +169,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -209,6 +212,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -282,6 +286,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -327,6 +332,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -401,6 +407,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -443,6 +450,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json index fc55b4cd..4df81578 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json @@ -43,6 +43,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -106,6 +107,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -155,6 +157,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -222,6 +225,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -271,6 +275,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json index ede8dd00..e2037e26 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json @@ -40,6 +40,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -71,6 +72,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -126,6 +128,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -157,6 +160,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -207,6 +211,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -270,6 +275,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -301,6 +307,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json index f44e69dd..5b129d7d 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -68,6 +69,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -109,6 +111,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -147,6 +150,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -185,6 +189,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -226,6 +231,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index 049722e1..a676dc6e 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -49,6 +49,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -117,6 +118,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -188,6 +190,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json index 536797ac..cc6d1ada 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json @@ -39,6 +39,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -96,6 +97,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -153,6 +155,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -210,6 +213,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -276,6 +280,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -346,6 +351,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -415,6 +421,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -472,6 +479,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -526,6 +534,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -580,6 +589,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -634,6 +644,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -691,6 +702,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json index 0d4ee88f..ea1363e7 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json @@ -55,6 +55,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -96,6 +97,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -167,6 +169,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -209,6 +212,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -282,6 +286,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -327,6 +332,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -401,6 +407,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -443,6 +450,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json index c49ac467..2c34edb4 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json @@ -42,6 +42,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -99,6 +100,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -142,6 +144,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -203,6 +206,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -246,6 +250,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json index f8bed840..368eb855 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json @@ -40,6 +40,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -70,6 +71,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -121,6 +123,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -152,6 +155,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -202,6 +206,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -265,6 +270,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -295,6 +301,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] }, diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json index 9a964aad..9393c4b3 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json @@ -26,6 +26,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -62,6 +63,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] }, @@ -98,6 +100,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -131,6 +134,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -164,6 +168,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -200,6 +205,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index cc0d88c9..6c46fee8 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -41,6 +41,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -104,6 +105,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -170,6 +172,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json index 2a1c799e..86960cc9 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json @@ -33,6 +33,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -86,6 +87,7 @@ "--prefix=../../bin/x86_64-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -139,6 +141,7 @@ "--prefix=../../bin/x86_64-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -193,6 +196,7 @@ "--prefix=../../bin/armv7m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -245,6 +249,7 @@ "--prefix=../../bin/armv7m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -298,6 +303,7 @@ "--prefix=../../bin/armv7m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -351,6 +357,7 @@ "--prefix=../../bin/armv8m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -403,6 +410,7 @@ "--prefix=../../bin/armv8m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -456,6 +464,7 @@ "--prefix=../../bin/armv8m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json index 182e692e..69af166d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json @@ -31,6 +31,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -82,6 +83,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -133,6 +135,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -185,6 +188,7 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-linux-gnu-", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -235,6 +239,7 @@ "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -286,6 +291,7 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-win-gnu-", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -337,6 +343,7 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-linux-gnu-", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -387,6 +394,7 @@ "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -438,6 +446,7 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-win-gnu-", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json index c5ff7dae..afc79511 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json @@ -31,6 +31,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -85,6 +86,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -139,6 +141,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -193,6 +196,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -256,6 +260,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -322,6 +327,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -383,6 +389,7 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -437,6 +444,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -488,6 +496,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -539,6 +548,7 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -590,6 +600,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -644,6 +655,7 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json index 02b873f3..f6c9f7cb 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json @@ -34,6 +34,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -88,6 +89,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -142,6 +144,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -196,6 +199,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -250,6 +254,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -306,6 +311,7 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -359,6 +365,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -412,6 +419,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json index bb1142c9..fbe038e7 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json @@ -49,6 +49,7 @@ "-Woverride-init", "-Wunsafe-loop-optimizations", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -102,6 +103,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-#warnings", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -155,6 +157,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-error=uninitialized", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -208,6 +211,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-someflag", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json index 4d69954b..f2ccadb9 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -63,6 +64,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -117,6 +119,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -164,6 +167,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -218,6 +222,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -273,6 +278,7 @@ "--prefix=../../bin/armv7a-cros-linux-gnueabihf-", "-D__KERNEL__", "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -324,6 +330,7 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json index 469678ab..dc641c96 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json @@ -46,6 +46,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -78,6 +79,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -140,6 +142,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -173,6 +176,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -237,6 +241,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -273,6 +278,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -338,6 +344,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -371,6 +378,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json index 665fddbb..54b994cc 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json @@ -34,6 +34,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -88,6 +89,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -128,6 +130,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -186,6 +189,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -226,6 +230,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json index af220315..5234715d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json @@ -31,6 +31,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -57,6 +58,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -103,6 +105,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -129,6 +132,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -170,6 +174,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -224,6 +229,7 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", @@ -250,6 +256,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json index 5bb170bd..24f90fbf 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -55,6 +56,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -88,6 +90,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -121,7 +124,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -153,7 +157,8 @@ "-Wno-deprecated-declarations", "-Wtrampolines", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -186,7 +191,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -219,7 +225,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -251,7 +258,8 @@ "-Wno-deprecated-declarations", "-Wtrampolines", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -284,7 +292,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json index 00f2a364..b2662930 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -58,6 +59,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -94,6 +96,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -127,6 +130,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -160,6 +164,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -196,6 +201,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json index 10ff7383..79e02c3d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json @@ -25,6 +25,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -61,6 +62,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -97,6 +99,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -133,6 +136,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -168,6 +172,7 @@ "-static-libgcc", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -205,6 +210,7 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -240,6 +246,7 @@ "-static-libgcc", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -275,6 +282,7 @@ "-static-libgcc", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json index 5228e12b..e0fe5515 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json @@ -24,6 +24,7 @@ "-static-libgcc", "-march=goldmont", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -59,6 +60,7 @@ "-static-libgcc", "-march=goldmont-plus", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -94,6 +96,7 @@ "-static-libgcc", "-march=skylake", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json index 23109bc4..ba2f292c 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -54,6 +55,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -90,6 +92,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -120,6 +123,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -156,6 +160,7 @@ "-fno-stack-protector", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -192,7 +197,8 @@ "-mthumb", "-fno-stack-protector", "-D__KERNEL__", - "main.cc" + "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -226,6 +232,7 @@ "-static-libgcc", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-mno-movbe" ], "env_updates": [ -- cgit v1.2.3 From 443e0b362f02045134713114bb34a4afad8090da Mon Sep 17 00:00:00 2001 From: George Burgess IV Date: Thu, 25 Nov 2021 14:58:43 -0800 Subject: cros_utils: Add a bug reporting module This lets us conveniently report bugs/cronjob status/etc. BUG=b:202424935, b:202425245 TEST=Unit tests Change-Id: Ic276f6ecc69a4c8c0088143177b17836cee70f97 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3319353 Reviewed-by: Jordan Abrahams Commit-Queue: George Burgess Tested-by: George Burgess --- cros_utils/bugs.py | 104 ++++++++++++++++++++++++++++++++++++++++ cros_utils/bugs_test.py | 124 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100755 cros_utils/bugs.py create mode 100755 cros_utils/bugs_test.py diff --git a/cros_utils/bugs.py b/cros_utils/bugs.py new file mode 100755 index 00000000..88fb7675 --- /dev/null +++ b/cros_utils/bugs.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Utilities to file bugs.""" + +import base64 +import datetime +import enum +import json +import os +from typing import Any, Dict, List, Optional + +X20_PATH = '/google/data/rw/teams/c-compiler-chrome/prod_bugs' + + +class WellKnownComponents(enum.IntEnum): + """A listing of "well-known" components recognized by our infra.""" + CrOSToolchainPublic = -1 + CrOSToolchainPrivate = -2 + + +def _WriteBugJSONFile(object_type: str, json_object: Dict[str, Any]): + """Writes a JSON file to X20_PATH with the given bug-ish object.""" + final_object = { + 'type': object_type, + 'value': json_object, + } + + # The name of this has two parts: + # - An easily sortable time, to provide uniqueness and let our service send + # things in the order they were put into the outbox. + # - 64 bits of entropy, so two racing bug writes don't clobber the same file. + now = datetime.datetime.utcnow().isoformat('T', 'seconds') + 'Z' + entropy = base64.urlsafe_b64encode(os.getrandom(8)) + entropy_str = entropy.rstrip(b'=').decode('utf-8') + file_path = os.path.join(X20_PATH, f'{now}_{entropy_str}.json') + + temp_path = file_path + '.in_progress' + try: + with open(temp_path, 'w') as f: + json.dump(final_object, f) + os.rename(temp_path, file_path) + except: + os.remove(temp_path) + raise + return file_path + + +def AppendToExistingBug(bug_id: int, body: str): + """Sends a reply to an existing bug.""" + _WriteBugJSONFile('AppendToExistingBugRequest', { + 'body': body, + 'bug_id': bug_id, + }) + + +def CreateNewBug(component_id: int, + title: str, + body: str, + assignee: Optional[str] = None, + cc: Optional[List[str]] = None): + """Sends a request to create a new bug. + + Args: + component_id: The component ID to add. Anything from WellKnownComponents + also works. + title: Title of the bug. Must be nonempty. + body: Body of the bug. Must be nonempty. + assignee: Assignee of the bug. Must be either an email address, or a + "well-known" assignee (detective, mage). + cc: A list of emails to add to the CC list. Must either be an email + address, or a "well-known" individual (detective, mage). + """ + obj = { + 'component_id': component_id, + 'subject': title, + 'body': body, + } + + if assignee: + obj['assignee'] = assignee + + if cc: + obj['cc'] = cc + + _WriteBugJSONFile('FileNewBugRequest', obj) + + +def SendCronjobLog(cronjob_name: str, failed: bool, message: str): + """Sends the record of a cronjob to our bug infra. + + cronjob_name: The name of the cronjob. Expected to remain consistent over + time. + failed: Whether the job failed or not. + message: Any seemingly relevant context. This is pasted verbatim in a bug, if + the cronjob infra deems it worthy. + """ + _WriteBugJSONFile('ChrotomationCronjobUpdate', { + 'name': cronjob_name, + 'message': message, + 'failed': failed, + }) diff --git a/cros_utils/bugs_test.py b/cros_utils/bugs_test.py new file mode 100755 index 00000000..03dee64d --- /dev/null +++ b/cros_utils/bugs_test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# We're testing protected methods, so allow protected access. +# pylint: disable=protected-access + +"""Tests bug filing bits.""" + +import json +import tempfile +import unittest +from unittest.mock import patch + +import bugs + + +class Tests(unittest.TestCase): + """Tests for the bugs module.""" + def testWritingJSONFileSeemsToWork(self): + """Tests JSON file writing.""" + old_x20_path = bugs.X20_PATH + + def restore_x20_path(): + bugs.X20_PATH = old_x20_path + + self.addCleanup(restore_x20_path) + + with tempfile.TemporaryDirectory() as tempdir: + bugs.X20_PATH = tempdir + file_path = bugs._WriteBugJSONFile( + 'ObjectType', { + 'foo': 'bar', + 'baz': bugs.WellKnownComponents.CrOSToolchainPublic, + }) + + self.assertTrue(file_path.startswith(tempdir), + f'Expected {file_path} to start with {tempdir}') + + with open(file_path) as f: + self.assertEqual( + json.load(f), + { + 'type': 'ObjectType', + 'value': { + 'foo': 'bar', + 'baz': int(bugs.WellKnownComponents.CrOSToolchainPublic), + }, + }, + ) + + @patch('bugs._WriteBugJSONFile') + def testAppendingToBugsSeemsToWork(self, mock_write_json_file): + """Tests AppendToExistingBug.""" + bugs.AppendToExistingBug(1234, 'hello, world!') + mock_write_json_file.assert_called_once_with( + 'AppendToExistingBugRequest', + { + 'body': 'hello, world!', + 'bug_id': 1234, + }, + ) + + @patch('bugs._WriteBugJSONFile') + def testBugCreationSeemsToWork(self, mock_write_json_file): + """Tests CreateNewBug.""" + test_case_additions = ( + {}, + { + 'component_id': bugs.WellKnownComponents.CrOSToolchainPublic, + }, + { + 'assignee': 'foo@gbiv.com', + 'cc': ['bar@baz.com'], + }, + ) + + for additions in test_case_additions: + test_case = { + 'component_id': 123, + 'title': 'foo', + 'body': 'bar', + **additions, + } + + bugs.CreateNewBug(**test_case) + + expected_output = { + 'component_id': test_case['component_id'], + 'subject': test_case['title'], + 'body': test_case['body'], + } + + assignee = test_case.get('assignee') + if assignee: + expected_output['assignee'] = assignee + + cc = test_case.get('cc') + if cc: + expected_output['cc'] = cc + + mock_write_json_file.assert_called_once_with( + 'FileNewBugRequest', + expected_output, + ) + mock_write_json_file.reset_mock() + + @patch('bugs._WriteBugJSONFile') + def testCronjobLogSendingSeemsToWork(self, mock_write_json_file): + """Tests SendCronjobLog.""" + bugs.SendCronjobLog('my_name', False, 'hello, world!') + mock_write_json_file.assert_called_once_with( + 'ChrotomationCronjobUpdate', + { + 'name': 'my_name', + 'message': 'hello, world!', + 'failed': False, + }, + ) + + +if __name__ == '__main__': + unittest.main() -- cgit v1.2.3 From e8d03f7ea49a9ee46d672ed9ad3582ab591fd9f1 Mon Sep 17 00:00:00 2001 From: George Burgess IV Date: Thu, 9 Dec 2021 00:28:14 -0800 Subject: rust_watch: report bugs for new minor or major versions of rustc Now that we have new bug reporting bits, using them seems cool. File bugs for uprevving to each new (non-patch) version, so we no longer have to. BUG=None TEST=Unittests Change-Id: I2b458cfd92bcdfda2f4bd8fbac319c1d05c44b47 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3324374 Reviewed-by: Bob Haarman Commit-Queue: George Burgess Tested-by: George Burgess --- rust_tools/rust_watch.py | 110 +++++++++++++++++++++++++++--------------- rust_tools/rust_watch_test.py | 107 ++++++++++++++-------------------------- 2 files changed, 107 insertions(+), 110 deletions(-) diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index b2c94acd..db6ae71b 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -19,10 +19,9 @@ import shutil import subprocess import sys import time -from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple -from cros_utils import email_sender -from cros_utils import tiny_render +from cros_utils import bugs, email_sender, tiny_render def gentoo_sha_to_link(sha: str) -> str: @@ -223,43 +222,63 @@ def atomically_write_state(state_file: pathlib.Path, state: State) -> None: temp_file.rename(state_file) -def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, - new_gentoo_commits: List[GitCommit] - ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: +def file_bug(title: str, body: str) -> None: + """Files a bug against gbiv@ with the given title/body.""" + bugs.CreateNewBug( + bugs.WellKnownComponents.CrOSToolchainPublic, + title, + body, + # To either take or reassign depending on the rotation. + assignee='gbiv@google.com', + ) + + +def maybe_compose_bug( + old_state: State, + newest_release: RustReleaseVersion, +) -> Optional[Tuple[str, str]]: + """Creates a bug to file about the new release, if doing is desired.""" + if newest_release == old_state.last_seen_release: + return None + + title = f'New rustc release detected: v{newest_release}' + body = ('A new release has been detected; we should probably roll to it. ' + "Please see go/crostc-rust-rotation for who's turn it is.") + return title, body + + +def maybe_compose_email( + new_gentoo_commits: List[GitCommit] +) -> Optional[Tuple[str, List[tiny_render.Piece]]]: """Creates an email given our new state, if doing so is appropriate.""" + if not new_gentoo_commits: + return None + subject_pieces = [] body_pieces = [] - if newest_release > old_state.last_seen_release: - subject_pieces.append('new rustc release detected') - body_pieces.append(f'Rustc tag for v{newest_release} was found.') + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] - if new_gentoo_commits: - # Separate the sections a bit for prettier output. - if body_pieces: - body_pieces += [tiny_render.line_break, tiny_render.line_break] + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) - if len(new_gentoo_commits) == 1: - subject_pieces.append('new rust ebuild commit detected') - body_pieces.append('commit:') - else: - subject_pieces.append('new rust ebuild commits detected') - body_pieces.append('commits (newest first):') - - commit_lines = [] - for commit in new_gentoo_commits: - commit_lines.append([ - tiny_render.Link( - gentoo_sha_to_link(commit.sha), - commit.sha[:12], - ), - f': {commit.subject}', - ]) - - body_pieces.append(tiny_render.UnorderedList(commit_lines)) - - if not subject_pieces: - return None + body_pieces.append(tiny_render.UnorderedList(commit_lines)) subject = '[rust-watch] ' + '; '.join(subject_pieces) return subject, body_pieces @@ -274,9 +293,9 @@ def main(argv: List[str]) -> None: parser.add_argument('--state_dir', required=True, help='Directory to store state in.') - parser.add_argument('--skip_email', + parser.add_argument('--skip_side_effects', action='store_true', - help="Don't send an email.") + help="Don't send an email or file a bug.") parser.add_argument( '--skip_state_update', action='store_true', @@ -319,14 +338,25 @@ def main(argv: List[str]) -> None: prior_state.last_gentoo_sha) logging.info('New commits: %r', new_commits) - maybe_email = maybe_compose_email(prior_state, most_recent_release, - new_commits) + maybe_bug = maybe_compose_bug(prior_state, most_recent_release) + maybe_email = maybe_compose_email(new_commits) + + if maybe_bug is None: + logging.info('No bug to file') + else: + title, body = maybe_bug + if opts.skip_side_effects: + logging.info('Skipping sending bug with title %r and contents\n%s', + title, body) + else: + logging.info('Writing new bug') + file_bug(title, body) if maybe_email is None: - logging.info('No updates to send') + logging.info('No email to send') else: title, body = maybe_email - if opts.skip_email: + if opts.skip_side_effects: logging.info('Skipping sending email with title %r and contents\n%s', title, tiny_render.render_html_pieces(body)) else: diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index a00f3ddc..30bacbb9 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -13,16 +13,15 @@ import time import unittest import unittest.mock -import rust_watch from cros_utils import tiny_render +import rust_watch + class Test(unittest.TestCase): """Tests.""" - def _silence_logs(self): """Silences all log output until the end of the current test.""" - def should_log(_record): return 0 @@ -38,8 +37,8 @@ class Test(unittest.TestCase): def test_release_version_json_round_trips(self): ver = rust_watch.RustReleaseVersion(1, 2, 3) - self.assertEqual( - rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver) + self.assertEqual(rust_watch.RustReleaseVersion.from_json(ver.to_json()), + ver) def test_state_json_round_trips(self): state = rust_watch.State( @@ -96,34 +95,14 @@ class Test(unittest.TestCase): rust_watch.GitCommit('abc123', 'newer commit'), ]) - def test_compose_email_on_a_new_release(self): - new_release = rust_watch.maybe_compose_email( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 1, 0), - new_gentoo_commits=[], - ) - - self.assertEqual(new_release, ('[rust-watch] new rustc release detected', - ['Rustc tag for v1.1.0 was found.'])) - def test_compose_email_on_a_new_gentoo_commit(self): sha_a = 'a' * 40 - new_commit = rust_watch.maybe_compose_email( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', + new_commit = rust_watch.maybe_compose_email(new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - new_gentoo_commits=[ - rust_watch.GitCommit( - sha=sha_a, - subject='summary_a', - ), - ], - ) + ], ) self.assertEqual(new_commit, ('[rust-watch] new rust ebuild commit detected', [ @@ -139,60 +118,48 @@ class Test(unittest.TestCase): ]) ])) - def test_compose_email_on_multiple_events(self): - sha_a = 'a' * 40 - new_commit_and_release = rust_watch.maybe_compose_email( + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) + + def test_compose_bug_creates_bugs_on_new_versions(self): + title, body = rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 1), + ) + self.assertEqual(title, 'New rustc release detected: v1.0.1') + self.assertTrue(body.startswith('A new release has been detected;')) + + title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), last_gentoo_sha='', ), newest_release=rust_watch.RustReleaseVersion(1, 1, 0), - new_gentoo_commits=[ - rust_watch.GitCommit( - sha=sha_a, - subject='summary_a', - ), - ], ) + self.assertEqual(title, 'New rustc release detected: v1.1.0') + self.assertTrue(body.startswith('A new release has been detected;')) - self.assertEqual( - new_commit_and_release, - ('[rust-watch] new rustc release detected; new rust ebuild commit ' - 'detected', [ - 'Rustc tag for v1.1.0 was found.', - tiny_render.line_break, - tiny_render.line_break, - 'commit:', - tiny_render.UnorderedList([ - [ - tiny_render.Link( - rust_watch.gentoo_sha_to_link(sha_a), - sha_a[:12], - ), - ': summary_a', - ], - ]), - ])) + title, body = rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(2, 0, 0), + ) + self.assertEqual(title, 'New rustc release detected: v2.0.0') + self.assertTrue(body.startswith('A new release has been detected;')) - def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + def test_compose_bug_does_nothing_when_no_new_updates_exist(self): self.assertIsNone( - rust_watch.maybe_compose_email( + rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), last_gentoo_sha='', ), newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - new_gentoo_commits=[], - )) - - self.assertIsNone( - rust_watch.maybe_compose_email( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - new_gentoo_commits=[], )) -- cgit v1.2.3 From ed2aa295b315c2d2570e942e6a056c45028f39b4 Mon Sep 17 00:00:00 2001 From: Manoj Gupta Date: Mon, 6 Dec 2021 11:46:42 -0800 Subject: compiler_wrapper: Drop "-fno-experimental-pass-manager" new pass manager is already default in upstream llvm and fixes have been made to broken features. So stop using the old pass manager with sanitizers. BUG=b:210661138 TEST=CQ Change-Id: Ia73f6c89a095a5cac4d8d175afd6ff18edd200a2 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3319340 Reviewed-by: Ryan Beltran Reviewed-by: Christopher Di Bella Commit-Queue: Manoj Gupta Tested-by: Manoj Gupta --- compiler_wrapper/sanitizer_flags.go | 17 ------- compiler_wrapper/sanitizer_flags_test.go | 52 ---------------------- .../clang_sanitizer_args.json | 2 - .../cros_hardened_golden/clang_sanitizer_args.json | 2 - .../clang_sanitizer_args.json | 2 - 5 files changed, 75 deletions(-) diff --git a/compiler_wrapper/sanitizer_flags.go b/compiler_wrapper/sanitizer_flags.go index abb385a8..da0a64b3 100644 --- a/compiler_wrapper/sanitizer_flags.go +++ b/compiler_wrapper/sanitizer_flags.go @@ -9,20 +9,13 @@ import ( ) func processSanitizerFlags(builder *commandBuilder) { - hasCoverageFlags := false hasSanitizeFlags := false - hasSanitizeFuzzerFlags := false for _, arg := range builder.args { // TODO: This should probably be -fsanitize= to not match on // e.g. -fsanitize-blocklist if arg.fromUser { if strings.HasPrefix(arg.value, "-fsanitize") { hasSanitizeFlags = true - if strings.Contains(arg.value, "fuzzer") { - hasSanitizeFuzzerFlags = true - } - } else if arg.value == "-fprofile-instr-generate" { - hasCoverageFlags = true } } } @@ -44,15 +37,5 @@ func processSanitizerFlags(builder *commandBuilder) { } return arg.value }) - if builder.target.compilerType == clangType { - // hasSanitizeFlags && hasCoverageFlags is to work around crbug.com/1013622 - if hasSanitizeFuzzerFlags || (hasSanitizeFlags && hasCoverageFlags) { - fuzzerFlagsToAdd := []string{ - // TODO: This flag should be removed once fuzzer works with new pass manager - "-fno-experimental-new-pass-manager", - } - builder.addPreUserArgs(fuzzerFlagsToAdd...) - } - } } } diff --git a/compiler_wrapper/sanitizer_flags_test.go b/compiler_wrapper/sanitizer_flags_test.go index 8f50a900..a401d58e 100644 --- a/compiler_wrapper/sanitizer_flags_test.go +++ b/compiler_wrapper/sanitizer_flags_test.go @@ -98,55 +98,3 @@ func TestKeepSanitizerFlagsIfSanitizeGivenInCommonFlags(t *testing.T) { } }) } - -func TestAddFuzzerFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fsanitize=fuzzer", mainCc))) - if err := verifyArgOrder(cmd, "-fno-experimental-new-pass-manager", - "-fsanitize=fuzzer", mainCc); err != nil { - t.Error(err) - } - }) -} - -func TestOmitFuzzerFlagsForGcc(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(gccX86_64, "-fsanitize=fuzzer", mainCc))) - if err := verifyArgCount(cmd, 0, "-fno-experimental-new-pass-manager"); err != nil { - t.Error(err) - } - }) -} - -func TestAddSanitizerCoverageFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fsanitize=address", "-fprofile-instr-generate", mainCc))) - if err := verifyArgOrder(cmd, "-fno-experimental-new-pass-manager", - "-fsanitize=address", "-fprofile-instr-generate", mainCc); err != nil { - t.Error(err) - } - }) -} - -func TestOmitSanitizerCoverageFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fsanitize=address", mainCc))) - if err := verifyArgCount(cmd, 0, "-fno-experimental-new-pass-manager"); err != nil { - t.Error(err) - } - }) -} - -func TestKeepSanitizerCoverageFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fprofile-instr-generate", mainCc))) - if err := verifyArgCount(cmd, 0, "-fno-experimental-new-pass-manager"); err != nil { - t.Error(err) - } - }) -} diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json index cc64bad9..b8b28cd9 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json @@ -206,7 +206,6 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", - "-fno-experimental-new-pass-manager", "-fsanitize=fuzzer", "main.cc", "-Wno-implicit-int-float-conversion", @@ -251,7 +250,6 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", - "-fno-experimental-new-pass-manager", "-fsanitize=address", "-fprofile-instr-generate", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json index 948c19b2..b71a8805 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json @@ -290,7 +290,6 @@ "-pie", "-fno-omit-frame-pointer", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=fuzzer", "main.cc", @@ -354,7 +353,6 @@ "-pie", "-fno-omit-frame-pointer", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "-fprofile-instr-generate", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json index f6c9f7cb..3e140d59 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json @@ -250,7 +250,6 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=fuzzer", "main.cc", @@ -306,7 +305,6 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "-fprofile-instr-generate", -- cgit v1.2.3 From 811429e30bbd53c9b0f4ec8c6b882f68d42588b9 Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Mon, 20 Dec 2021 23:44:04 +0000 Subject: llvm_tools: Catch dup SHAs in get_upstream_patch If a duplicate SHA is listed in get_upstream_patch.py, we need to report this as an error and not do anything. This commit adds this functionality, as well as some structures to make adding this feature easier. BUG=b:187795686 TEST=get_upstream_patch.py $args --sha $SHA --sha $SHA Change-Id: I56f1159d5bdd34c52efc00e991b7f1333b7ba3f7 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3351166 Commit-Queue: Christopher Di Bella Reviewed-by: Christopher Di Bella Commit-Queue: Jordan Abrahams Tested-by: Jordan Abrahams --- llvm_tools/get_upstream_patch.py | 122 +++++++++++++++++++++++++++------------ 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index f72891e0..ee207f17 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -6,8 +6,6 @@ """Get an upstream patch to LLVM's PATCHES.json.""" -from __future__ import print_function - import argparse import json import logging @@ -18,6 +16,8 @@ import sys import typing as t from datetime import datetime +import dataclasses + import chroot import get_llvm_hash import git @@ -219,33 +219,29 @@ def find_patches_and_make_cl( llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, create_cl: bool, skip_dependencies: bool, reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]]): + + converted_patches = [ + _convert_patch(llvm_config, skip_dependencies, p) for p in patches + ] + potential_duplicates = _get_duplicate_shas(converted_patches) + if potential_duplicates: + err_msg = '\n'.join(f'{a.patch} == {b.patch}' + for a, b in potential_duplicates) + raise RuntimeError(f'Found Duplicate SHAs:\n{err_msg}') + + # CL Related variables, only used if `create_cl` + symlinks_to_uprev = [] + commit_messages = [ + 'llvm: get patches from upstream\n', + ] + branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' + if create_cl: - branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' git.CreateBranch(llvm_symlink_dir, branch) - symlinks_to_uprev = [] - commit_messages = [ - 'llvm: get patches from upstream\n', - ] - - for patch in patches: - # git hash should only have lower-case letters - is_differential = patch.startswith('D') - if is_differential: - subprocess.check_output( - [ - 'arc', 'patch', '--nobranch', - '--skip-dependencies' if skip_dependencies else '--revision', - patch - ], - cwd=llvm_config.dir, - ) - sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') - rev = patch - else: - sha = resolve_llvm_ref(llvm_config.dir, patch) - rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + + for parsed_patch in converted_patches: # Find out the llvm projects changed in this commit - packages = get_package_names(sha, llvm_config.dir) + packages = get_package_names(parsed_patch.sha, llvm_config.dir) # Find out the ebuild symlinks of the corresponding ChromeOS packages symlinks = chroot.GetChrootEbuildPaths(chroot_path, [ 'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package @@ -253,23 +249,20 @@ def find_patches_and_make_cl( ]) symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks) # Create a local patch for all the affected llvm projects - create_patch_for_packages(packages, symlinks, start_rev, rev, sha, - llvm_config.dir) + create_patch_for_packages(packages, symlinks, start_rev, parsed_patch.rev, + parsed_patch.sha, llvm_config.dir) if create_cl: symlinks_to_uprev.extend(symlinks) - if is_differential: - msg = f'\n\nreviews.llvm.org/{patch}\n' - else: - msg = f'\n\nreviews.llvm.org/rG{sha}\n' commit_messages.extend([ - msg, - subprocess.check_output(['git', 'log', '-n1', '--oneline', sha], - cwd=llvm_config.dir, - encoding='utf-8') + parsed_patch.git_msg(), + subprocess.check_output( + ['git', 'log', '-n1', '--oneline', parsed_patch.sha], + cwd=llvm_config.dir, + encoding='utf-8') ]) - if is_differential: + if parsed_patch.is_differential: subprocess.check_output(['git', 'reset', '--hard', 'HEAD^'], cwd=llvm_config.dir) @@ -278,6 +271,61 @@ def find_patches_and_make_cl( reviewers, cc) +@dataclasses.dataclass(frozen=True) +class ParsedPatch: + """Class to keep track of bundled patch info.""" + patch: str + sha: str + is_differential: bool + rev: t.Union[git_llvm_rev.Rev, str] + + def git_msg(self) -> str: + if self.is_differential: + return f'\n\nreviews.llvm.org/{self.patch}\n' + return f'\n\nreviews.llvm.org/rG{self.sha}\n' + + +def _convert_patch(llvm_config: git_llvm_rev.LLVMConfig, + skip_dependencies: bool, patch: str) -> ParsedPatch: + """Extract git revision info from a patch. + + Args: + llvm_config: LLVM configuration object. + skip_dependencies: Pass --skip-dependecies for to `arc` + patch: A single patch referent string. + + Returns: + A [ParsedPatch] object. + """ + + # git hash should only have lower-case letters + is_differential = patch.startswith('D') + if is_differential: + subprocess.check_output( + [ + 'arc', 'patch', '--nobranch', + '--skip-dependencies' if skip_dependencies else '--revision', patch + ], + cwd=llvm_config.dir, + ) + sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') + rev = patch + else: + sha = resolve_llvm_ref(llvm_config.dir, patch) + rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + return ParsedPatch(patch=patch, + sha=sha, + rev=rev, + is_differential=is_differential) + + +def _get_duplicate_shas( + patches: t.List[ParsedPatch]) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]: + """Return a list of Patches which have duplicate SHA's""" + return [(left, right) for i, left in enumerate(patches) + for right in patches[i + 1:] if left.sha == right.sha] + + def get_from_upstream(chroot_path: str, create_cl: bool, start_sha: str, -- cgit v1.2.3 From 26b91865af26f397c22d7d403d01dbb38d3510bd Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Tue, 21 Dec 2021 19:51:23 +0000 Subject: llvm_tools: Update get_upstream_patch schema The get_upstream_patch.py program is using the old schema for PATCHES.json, and appending with that old schema to the new format. This commit fixes this issue. Additionally, this ensures that the keys are sorted on write to prevent line-diffs from getting out of sync. BUG=b:209493133 TEST=get_upstream_patch $args --platform chromiumos --platform android Change-Id: I0f96843b0134daa27574a94f583efdd27f394a3f Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3352574 Reviewed-by: Christopher Di Bella Commit-Queue: Jordan Abrahams Tested-by: Jordan Abrahams --- llvm_tools/get_upstream_patch.py | 72 +++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index ee207f17..7a4be3eb 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -24,6 +24,12 @@ import git import git_llvm_rev import update_chromeos_llvm_hash +__DOC_EPILOGUE = """ +Example Usage: + get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \ +--sha 1234567 --sha 890abdc +""" + class CherrypickError(ValueError): """A ValueError that highlights the cherry-pick has been seen before""" @@ -31,8 +37,8 @@ class CherrypickError(ValueError): def add_patch(patches_json_path: str, patches_dir: str, relative_patches_dir: str, start_version: git_llvm_rev.Rev, - llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, - str], sha: str, package: str): + llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str, + package: str, platforms: t.List[str]): """Gets the start and end intervals in 'json_file'. Args: @@ -47,6 +53,7 @@ def add_patch(patches_json_path: str, patches_dir: str, revisions, the git sha from the local commit created by 'arc patch' is used. package: The LLVM project name this patch applies to. + platforms: List of platforms this patch applies to. Raises: CherrypickError: A ValueError that highlights the cherry-pick has been @@ -89,18 +96,25 @@ def add_patch(patches_json_path: str, patches_dir: str, cwd=llvm_dir, encoding='utf-8') - patch_metadata = { - 'comment': commit_subject.strip(), + patch_props = { 'rel_patch_path': rel_patch_path, 'start_version': start_version.number, + 'metadata': { + 'title': commit_subject.strip(), + 'info': [], + }, + 'platforms': sorted(platforms), + 'end_version': rev.number if isinstance(rev, git_llvm_rev.Rev) else None, } - if isinstance(rev, git_llvm_rev.Rev): - patch_metadata['end_version'] = rev.number - patches_json.append(patch_metadata) + patches_json.append(patch_props) temp_file = patches_json_path + '.tmp' with open(temp_file, 'w', encoding='utf-8') as f: - json.dump(patches_json, f, indent=4, separators=(',', ': ')) + json.dump(patches_json, + f, + indent=4, + separators=(',', ': '), + sort_keys=True) f.write('\n') os.rename(temp_file, patches_json_path) @@ -179,8 +193,8 @@ def get_package_names(sha: str, llvm_dir: str) -> list: def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], start_rev: git_llvm_rev.Rev, - rev: t.Union[git_llvm_rev.Rev, - str], sha: str, llvm_dir: str): + rev: t.Union[git_llvm_rev.Rev, str], sha: str, + llvm_dir: str, platforms: t.List[str]): """Create a patch and add its metadata for each package""" for package, symlink in zip(packages, symlinks): symlink_dir = os.path.dirname(symlink) @@ -188,8 +202,15 @@ def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], relative_patches_dir = 'cherry' if package == 'llvm' else '' patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) logging.info('Getting %s (%s) into %s', rev, sha, package) - add_patch(patches_json_path, patches_dir, relative_patches_dir, start_rev, - llvm_dir, rev, sha, package) + add_patch(patches_json_path, + patches_dir, + relative_patches_dir, + start_rev, + llvm_dir, + rev, + sha, + package, + platforms=platforms) def make_cl(symlinks_to_uprev: t.List[str], llvm_symlink_dir: str, branch: str, @@ -218,7 +239,8 @@ def find_patches_and_make_cl( chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev, llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, create_cl: bool, skip_dependencies: bool, - reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]]): + reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]], + platforms: t.List[str]): converted_patches = [ _convert_patch(llvm_config, skip_dependencies, p) for p in patches @@ -249,8 +271,13 @@ def find_patches_and_make_cl( ]) symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks) # Create a local patch for all the affected llvm projects - create_patch_for_packages(packages, symlinks, start_rev, parsed_patch.rev, - parsed_patch.sha, llvm_config.dir) + create_patch_for_packages(packages, + symlinks, + start_rev, + parsed_patch.rev, + parsed_patch.sha, + llvm_config.dir, + platforms=platforms) if create_cl: symlinks_to_uprev.extend(symlinks) @@ -330,6 +357,7 @@ def get_from_upstream(chroot_path: str, create_cl: bool, start_sha: str, patches: t.List[str], + platforms: t.List[str], skip_dependencies: bool = False, reviewers: t.List[str] = None, cc: t.List[str] = None): @@ -355,6 +383,7 @@ def get_from_upstream(chroot_path: str, find_patches_and_make_cl(chroot_path=chroot_path, patches=patches, + platforms=platforms, start_rev=git_llvm_rev.translate_sha_to_rev( llvm_config, start_sha), llvm_config=llvm_config, @@ -373,7 +402,10 @@ def main(): level=logging.INFO, ) - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__DOC_EPILOGUE) parser.add_argument('--chroot_path', default=os.path.join(os.path.expanduser('~'), 'chromiumos'), @@ -392,6 +424,13 @@ def main(): action='append', default=[], help='The LLVM differential revision to apply. Example: D1234') + parser.add_argument( + '--platform', + action='append', + required=True, + help='Apply this patch to the give platform. Common options include ' + '"chromiumos" and "android". Can be specified multiple times to ' + 'apply to multiple platforms') parser.add_argument('--create_cl', action='store_true', help='Automatically create a CL if specified') @@ -415,6 +454,7 @@ def main(): start_sha=args.start_sha, patches=args.sha + args.differential, skip_dependencies=args.skip_dependencies, + platforms=args.platform, ) -- cgit v1.2.3 From d60f468ccc3b3be93b2b09051b474cd2df0e0856 Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Mon, 20 Dec 2021 11:08:49 -0800 Subject: afdo_metadata: Publish the new kernel profiles Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 Update chromeos-kernel-5_4 BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator Change-Id: I6ec43a9b06720771f3c05ae03021a2b05aac2387 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3350701 Tested-by: Denis Nikitin Auto-Submit: Denis Nikitin Reviewed-by: Jordan Abrahams Commit-Queue: Denis Nikitin --- afdo_metadata/kernel_afdo.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 98a00881..a3baf249 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,11 @@ { - "chromeos-kernel-4_4": { - "name": "R98-14324.19-1638181896" - }, "chromeos-kernel-4_14": { - "name": "R98-14350.0-1638182285" + "name": "R99-14385.0-1639392232" }, "chromeos-kernel-4_19": { - "name": "R98-14352.0-1638182035" + "name": "R99-14385.0-1639391543" }, "chromeos-kernel-5_4": { - "name": "R98-14352.0-1638181967" + "name": "R99-14382.0-1639392024" } } -- cgit v1.2.3 From a170d3e2fbe4a6126377c2cf3e9c348a1d981d03 Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Tue, 28 Dec 2021 22:26:10 +0000 Subject: crosperf: remove dead bob DUT The chromeos6-row3-rack13-host15.cros DUT has been removed and is pending replacement. This commit removes it from the default_remotes. This commit may be reverted when the DUT is replaced. BUG=b:210575915 TEST=None Change-Id: Ifc71364b40c92f6a3fd136d2191a79308aa98099 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3359639 Reviewed-by: Denis Nikitin Commit-Queue: Jordan Abrahams Tested-by: Jordan Abrahams --- crosperf/default_remotes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 1cc628f9..faecb833 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,4 +1,4 @@ -bob : chromeos6-row4-rack13-host6.cros chromeos6-row3-rack13-host15.cros +bob : chromeos6-row4-rack13-host6.cros chell : chromeos2-row1-rack10-host2.cros chromeos2-row1-rack10-host4.cros coral : chromeos6-row5-rack6-host1.cros chromeos6-row5-rack6-host3.cros chromeos6-row5-rack6-host5.cros elm : chromeos6-row14-rack15-host21.cros -- cgit v1.2.3 From ed486b7b3a81aa47764e5b59c27c2f07e0d8e50f Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Wed, 29 Dec 2021 13:33:07 -0800 Subject: afdo_metadata: Publish the new kernel profiles Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 Update chromeos-kernel-5_4 BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator Change-Id: I89ef7b462b627edbe5a0b73b58c58d1689f8c306 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3361355 Tested-by: Denis Nikitin Reviewed-by: Jordan Abrahams Commit-Queue: Denis Nikitin --- afdo_metadata/kernel_afdo.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index a3baf249..a3275004 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,11 +1,14 @@ { + "chromeos-kernel-4_4": { + "name": "R99-14388.8-1640601492" + }, "chromeos-kernel-4_14": { - "name": "R99-14385.0-1639392232" + "name": "R99-14407.0-1640601365" }, "chromeos-kernel-4_19": { - "name": "R99-14385.0-1639391543" + "name": "R99-14388.8-1640601108" }, "chromeos-kernel-5_4": { - "name": "R99-14382.0-1639392024" + "name": "R99-14397.0-1640601195" } } -- cgit v1.2.3 From 457d96b56c9979190b58e07bc1304ac53be49c4c Mon Sep 17 00:00:00 2001 From: George Burgess IV Date: Mon, 3 Jan 2022 16:00:24 -0800 Subject: pgo_tools: make monitor_pgo_profiles fail rather than emailing Prior to these changes, this script sends emails directly. These are... not as nice to work with as bugs. Since we run this through our new bug-filing cron bits, just make this script output to stdout/stderr & provide a meaningful exit code. Having bugs to track when these profiles are out-of-date is nicer than tracking N emails. BUG=None TEST=Unittests Change-Id: Id5f7838340df78006bddb36c3bc128016eb2acb5 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3358469 Tested-by: George Burgess Auto-Submit: George Burgess Reviewed-by: Denis Nikitin Commit-Queue: Denis Nikitin --- pgo_tools/monitor_pgo_profiles.py | 70 ++++++++-------------------- pgo_tools/monitor_pgo_profiles_unittest.py | 74 +++++++++++++----------------- 2 files changed, 52 insertions(+), 92 deletions(-) diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py index be159b94..5c17423b 100755 --- a/pgo_tools/monitor_pgo_profiles.py +++ b/pgo_tools/monitor_pgo_profiles.py @@ -7,14 +7,11 @@ import argparse import datetime -import sys -import subprocess import logging +import subprocess +import sys from typing import List, NamedTuple, Optional, Tuple -from cros_utils import email_sender -from cros_utils import tiny_render - PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/' 'pgo-generate-llvm-next-orchestrator') @@ -54,47 +51,35 @@ def fetch_most_recent_profdata(arch: str) -> ProfdataInfo: return max(infos) -def compose_complaint_email( +def compose_complaint( out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]] -) -> Optional[Tuple[str, tiny_render.Piece]]: +) -> Optional[str]: if not out_of_date_profiles: return None if len(out_of_date_profiles) == 1: - subject = '1 llvm profile is out of date' - body = ['out-of-date profile:'] + body_lines = ['1 profile is out of date:'] else: - subject = f'{len(out_of_date_profiles)} llvm profiles are out of date' - body = ['out-of-date profiles:'] + body_lines = [f'{len(out_of_date_profiles)} profiles are out of date:'] - out_of_date_items = [] for arch, profdata_info in out_of_date_profiles: - out_of_date_items.append( - f'{arch} (most recent profile was from {profdata_info.date} at ' + body_lines.append( + f'- {arch} (most recent profile was from {profdata_info.date} at ' f'{profdata_info.location!r})') - body += [ - tiny_render.UnorderedList(out_of_date_items), - tiny_render.line_break, - tiny_render.line_break, + body_lines.append('\n') + body_lines.append( 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' - 'Their status can be found at ', - tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK), - '.', - ] - return subject, body + f'Their status can be found at {PGO_BUILDBOT_LINK}.') + return '\n'.join(body_lines) def main() -> None: logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--dry_run', - action='store_true', - help="Don't actually send an email", - ) + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '--max_age_days', # These builders run ~weekly. If we fail to generate two in a row, @@ -119,29 +104,12 @@ def main() -> None: if age >= max_age: out_of_date_profiles.append((arch, most_recent)) - email = compose_complaint_email(out_of_date_profiles) - if not email: - logging.info('No email to send; quit') - return - - subject, body = email + complaint = compose_complaint(out_of_date_profiles) + if complaint: + logging.error('%s', complaint) + sys.exit(1) - identifier = 'llvm-pgo-monitor' - subject = f'[{identifier}] {subject}' - - logging.info('Sending email with title %r', subject) - if args.dry_run: - logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject, - tiny_render.render_text_pieces(body)) - else: - email_sender.EmailSender().SendX20Email( - subject=subject, - identifier=identifier, - well_known_recipients=['mage'], - direct_recipients=['gbiv@google.com'], - text_body=tiny_render.render_text_pieces(body), - html_body=tiny_render.render_html_pieces(body), - ) + logging.info('Nothing seems wrong') if __name__ == '__main__': diff --git a/pgo_tools/monitor_pgo_profiles_unittest.py b/pgo_tools/monitor_pgo_profiles_unittest.py index b4e085ec..eef33887 100755 --- a/pgo_tools/monitor_pgo_profiles_unittest.py +++ b/pgo_tools/monitor_pgo_profiles_unittest.py @@ -11,41 +11,36 @@ import unittest import unittest.mock import monitor_pgo_profiles -from cros_utils import tiny_render class Test(unittest.TestCase): """Tests for monitor_pgo_profiles.""" + def test_compose_complaint_with_zero_out_of_date(self): + self.assertIsNone(monitor_pgo_profiles.compose_complaint([])) - def test_compose_complaint_email_with_zero_out_of_date(self): - self.assertIsNone(monitor_pgo_profiles.compose_complaint_email([])) - - def test_compose_complaint_email_with_one_out_of_date(self): + def test_compose_complaint_with_one_out_of_date(self): profdata_info = monitor_pgo_profiles.ProfdataInfo( date=datetime.datetime(2020, 1, 2, 3, 4, 5), location='gs://somewhere', ) - result = monitor_pgo_profiles.compose_complaint_email([ + result = monitor_pgo_profiles.compose_complaint([ ('some_arch', profdata_info), ]) - self.assertEqual(result, ('1 llvm profile is out of date', [ - 'out-of-date profile:', - tiny_render.UnorderedList([ - f'some_arch (most recent profile was from {profdata_info.date} at ' - f'{profdata_info.location!r})' - ]), - tiny_render.line_break, - tiny_render.line_break, - 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' - 'Their status can be found at ', - tiny_render.Link( - href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - ), - '.', - ])) + self.assertEqual( + result, + '\n'.join(( + '1 profile is out of date:', + f'- some_arch (most recent profile was from {profdata_info.date} ' + f'at {profdata_info.location!r})', + '', + '', + 'PTAL to see if the llvm-pgo-generate bots are functioning ' + 'normally. Their status can be found at ' + f'{monitor_pgo_profiles.PGO_BUILDBOT_LINK}.', + )), + ) - def test_compose_complaint_email_with_two_out_of_date(self): + def test_compose_complaint_with_two_out_of_date(self): profdata_info_1 = monitor_pgo_profiles.ProfdataInfo( date=datetime.datetime(2020, 1, 2, 3, 4, 5), location='gs://somewhere', @@ -54,28 +49,25 @@ class Test(unittest.TestCase): date=datetime.datetime(2020, 3, 2, 1, 4, 5), location='gs://somewhere-else', ) - result = monitor_pgo_profiles.compose_complaint_email([ + result = monitor_pgo_profiles.compose_complaint([ ('some_arch', profdata_info_1), ('some_other_arch', profdata_info_2), ]) - self.assertEqual(result, ('2 llvm profiles are out of date', [ - 'out-of-date profiles:', - tiny_render.UnorderedList([ - f'some_arch (most recent profile was from {profdata_info_1.date} ' + self.assertEqual( + result, + '\n'.join(( + '2 profiles are out of date:', + f'- some_arch (most recent profile was from {profdata_info_1.date} ' f'at {profdata_info_1.location!r})', - f'some_other_arch (most recent profile was from ' - f'{profdata_info_2.date} at {profdata_info_2.location!r})' - ]), - tiny_render.line_break, - tiny_render.line_break, - 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' - 'Their status can be found at ', - tiny_render.Link( - href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - ), - '.', - ])) + f'- some_other_arch (most recent profile was from ' + f'{profdata_info_2.date} at {profdata_info_2.location!r})', + '', + '', + 'PTAL to see if the llvm-pgo-generate bots are functioning ' + 'normally. Their status can be found at ' + f'{monitor_pgo_profiles.PGO_BUILDBOT_LINK}.', + )), + ) @unittest.mock.patch.object(subprocess, 'run') def test_fetching_profdata_functions(self, subprocess_run_mock): -- cgit v1.2.3 From 72bb9a3507071b1f0c45c9b4a2656d6d0d7ba6fe Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Tue, 14 Dec 2021 21:41:25 +0000 Subject: patch_sync: Base code, version control managing This is the base code addition for the patch_sync project. This project attempts to synchronize LLVM patches and ChromeOS. This code is not meant to run, but simply be the starting point so that we break up the large code dump. This code is meant to compile, but is not meant to be run. It only adds the version control code which manages `repo` interactions. Additionally, this adds the full Cargo.toml (at least for the current patch stack). BUG=b:209493133 TEST=cargo build Change-Id: I903edba6e633ec53a7fa4d2e7b0c2aac1094933a Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3339404 Reviewed-by: George Burgess Tested-by: Jordan Abrahams Commit-Queue: Jordan Abrahams --- llvm_tools/patch_sync/.gitignore | 1 + llvm_tools/patch_sync/Cargo.lock | 453 +++++++++++++++++++++++++++ llvm_tools/patch_sync/Cargo.toml | 20 ++ llvm_tools/patch_sync/src/main.rs | 106 +++++++ llvm_tools/patch_sync/src/version_control.rs | 238 ++++++++++++++ 5 files changed, 818 insertions(+) create mode 100644 llvm_tools/patch_sync/.gitignore create mode 100644 llvm_tools/patch_sync/Cargo.lock create mode 100644 llvm_tools/patch_sync/Cargo.toml create mode 100644 llvm_tools/patch_sync/src/main.rs create mode 100644 llvm_tools/patch_sync/src/version_control.rs diff --git a/llvm_tools/patch_sync/.gitignore b/llvm_tools/patch_sync/.gitignore new file mode 100644 index 00000000..2f7896d1 --- /dev/null +++ b/llvm_tools/patch_sync/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/llvm_tools/patch_sync/Cargo.lock b/llvm_tools/patch_sync/Cargo.lock new file mode 100644 index 00000000..63a9fcf8 --- /dev/null +++ b/llvm_tools/patch_sync/Cargo.lock @@ -0,0 +1,453 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "patch_sync" +version = "0.1.0" +dependencies = [ + "anyhow", + "rand", + "regex", + "serde", + "serde_json", + "sha2", + "structopt", + "time", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f84e92c0f7c9d58328b85a78557813e4bd845130db68d7184635344399423b1" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "serde" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +dependencies = [ + "block-buffer", + "cfg-if", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a1dfb999630e338648c83e91c59a4e9fb7620f520c3194b6b89e276f2f1959" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "time" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41effe7cfa8af36f439fac33861b66b049edc6f9a32331e2312660529c1c24ad" +dependencies = [ + "libc", +] + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/llvm_tools/patch_sync/Cargo.toml b/llvm_tools/patch_sync/Cargo.toml new file mode 100644 index 00000000..43082627 --- /dev/null +++ b/llvm_tools/patch_sync/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "patch_sync" +version = "0.1.0" +authors = ["Jordan R Abrahams-Whitehead "] +edition = "2018" + +[profile.release] +panic = "abort" + +[dependencies] +anyhow = "1.0" +regex = "1.5" +serde = {version = "1.0", features = ["derive"]} +serde_json = "1.0" +sha2 = "0.9" +structopt = "0.3" +time = "0.3" + +[dev-dependencies] +rand = "0.8" diff --git a/llvm_tools/patch_sync/src/main.rs b/llvm_tools/patch_sync/src/main.rs new file mode 100644 index 00000000..8eedaaa8 --- /dev/null +++ b/llvm_tools/patch_sync/src/main.rs @@ -0,0 +1,106 @@ +mod version_control; + +use anyhow::{Context, Result}; +use std::path::PathBuf; +use structopt::StructOpt; + +fn main() -> Result<()> { + match Opt::from_args() { + Opt::Show { .. } => todo!("Opt show is not implemented yet"), + Opt::Transpose { + cros_checkout_path, + old_cros_ref, + android_checkout_path, + old_android_ref, + verbose, + dry_run, + no_commit, + } => transpose_subcmd(TransposeOpt { + cros_checkout_path, + old_cros_ref, + android_checkout_path, + old_android_ref, + verbose, + dry_run, + no_commit, + }), + } +} + +#[allow(dead_code)] +struct TransposeOpt { + cros_checkout_path: PathBuf, + old_cros_ref: String, + android_checkout_path: PathBuf, + old_android_ref: String, + verbose: bool, + dry_run: bool, + no_commit: bool, +} + +fn transpose_subcmd(args: TransposeOpt) -> Result<()> { + let ctx = version_control::RepoSetupContext { + cros_checkout: args.cros_checkout_path, + android_checkout: args.android_checkout_path, + sync_before: false, + }; + ctx.setup()?; + let _cros_patches_path = ctx.cros_patches_path(); + let _android_patches_path = ctx.android_patches_path(); + + if !args.no_commit { + return Ok(()); + } + // Commit and upload for review. + ctx.cros_repo_upload() + .context("uploading chromiumos changes")?; + ctx.android_repo_upload() + .context("uploading android changes")?; + Ok(()) +} + +#[derive(Debug, structopt::StructOpt)] +#[structopt(name = "patch_sync", about = "A pipeline for syncing the patch code")] +enum Opt { + /// Show a combined view of the PATCHES.json file, without making any changes. + #[allow(dead_code)] + Show { + #[structopt(parse(from_os_str))] + cros_checkout_path: PathBuf, + #[structopt(parse(from_os_str))] + android_checkout_path: PathBuf, + }, + /// Transpose patches from two PATCHES.json files + /// to each other. + Transpose { + #[structopt(long = "cros-checkout", parse(from_os_str))] + /// Path to the ChromiumOS source repo checkout. + cros_checkout_path: PathBuf, + + #[structopt(long = "overlay-base-ref")] + /// Git ref (e.g. hash) for the ChromiumOS overlay to use as the base. + old_cros_ref: String, + + #[structopt(long = "aosp-checkout", parse(from_os_str))] + /// Path to the Android Open Source Project source repo checkout. + android_checkout_path: PathBuf, + + #[structopt(long = "aosp-base-ref")] + /// Git ref (e.g. hash) for the llvm_android repo to use as the base. + old_android_ref: String, + + #[structopt(short, long)] + /// Print information to stdout + verbose: bool, + + #[structopt(long)] + /// Do not change any files. Useful in combination with `--verbose` + /// Implies `--no-commit` and `--no-upload`. + dry_run: bool, + + #[structopt(long)] + /// Do not commit any changes made. + /// Implies `--no-upload`. + no_commit: bool, + }, +} diff --git a/llvm_tools/patch_sync/src/version_control.rs b/llvm_tools/patch_sync/src/version_control.rs new file mode 100644 index 00000000..3dc5aae9 --- /dev/null +++ b/llvm_tools/patch_sync/src/version_control.rs @@ -0,0 +1,238 @@ +use anyhow::{anyhow, bail, ensure, Context, Result}; +use regex::Regex; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +const CHROMIUMOS_OVERLAY_REL_PATH: &str = "src/third_party/chromiumos-overlay"; +const ANDROID_LLVM_REL_PATH: &str = "toolchain/llvm_android"; + +/// Context struct to keep track of both Chromium OS and Android checkouts. +#[derive(Debug)] +pub struct RepoSetupContext { + pub cros_checkout: PathBuf, + pub android_checkout: PathBuf, + /// Run `repo sync` before doing any comparisons. + pub sync_before: bool, +} + +impl RepoSetupContext { + pub fn setup(&self) -> Result<()> { + if self.sync_before { + repo_cd_cmd(&self.cros_checkout, &["sync", CHROMIUMOS_OVERLAY_REL_PATH])?; + repo_cd_cmd(&self.android_checkout, &["sync", ANDROID_LLVM_REL_PATH])?; + } + Ok(()) + } + + pub fn cros_repo_upload(&self) -> Result<()> { + let llvm_dir = self + .cros_checkout + .join(&CHROMIUMOS_OVERLAY_REL_PATH) + .join("sys-devel/llvm"); + ensure!( + llvm_dir.is_dir(), + "CrOS LLVM dir {} is not a directory", + llvm_dir.display() + ); + Self::rev_bump_llvm(&llvm_dir)?; + Self::repo_upload( + &self.cros_checkout, + CHROMIUMOS_OVERLAY_REL_PATH, + &Self::build_commit_msg("android", "chromiumos", "BUG=None\nTEST=CQ"), + ) + } + + pub fn android_repo_upload(&self) -> Result<()> { + Self::repo_upload( + &self.android_checkout, + ANDROID_LLVM_REL_PATH, + &Self::build_commit_msg("chromiumos", "android", "Test: N/A"), + ) + } + + fn repo_upload(path: &Path, git_wd: &str, commit_msg: &str) -> Result<()> { + // TODO(ajordanr): Need to clean up if there's any failures during upload. + let git_path = &path.join(&git_wd); + ensure!( + git_path.is_dir(), + "git_path {} is not a directory", + git_path.display() + ); + repo_cd_cmd(path, &["start", "patch_sync_branch", git_wd])?; + git_cd_cmd(git_path, &["add", "."])?; + git_cd_cmd(git_path, &["commit", "-m", commit_msg])?; + repo_cd_cmd(path, &["upload", "-y", "--verify", git_wd])?; + Ok(()) + } + + pub fn android_patches_path(&self) -> PathBuf { + self.android_checkout + .join(&ANDROID_LLVM_REL_PATH) + .join("patches/PATCHES.json") + } + + pub fn cros_patches_path(&self) -> PathBuf { + self.cros_checkout + .join(&CHROMIUMOS_OVERLAY_REL_PATH) + .join("sys-devel/llvm/files/PATCHES.json") + } + + /// Increment LLVM's revision number + fn rev_bump_llvm(llvm_dir: &Path) -> Result { + let ebuild = find_ebuild(llvm_dir) + .with_context(|| format!("finding ebuild in {} to rev bump", llvm_dir.display()))?; + let ebuild_dir = ebuild.parent().unwrap(); + let suffix_matcher = Regex::new(r"-r([0-9]+)\.ebuild").unwrap(); + let ebuild_name = ebuild + .file_name() + .unwrap() + .to_str() + .ok_or_else(|| anyhow!("converting ebuild filename to utf-8"))?; + let new_path = if let Some(captures) = suffix_matcher.captures(ebuild_name) { + let full_suffix = captures.get(0).unwrap().as_str(); + let cur_version = captures.get(1).unwrap().as_str().parse::().unwrap(); + let new_filename = + ebuild_name.replace(full_suffix, &format!("-r{}.ebuild", cur_version + 1_u32)); + let new_path = ebuild_dir.join(new_filename); + fs::rename(&ebuild, &new_path)?; + new_path + } else { + // File did not end in a revision. We should append -r1 to the end. + let new_filename = ebuild.file_stem().unwrap().to_string_lossy() + "-r1.ebuild"; + let new_path = ebuild_dir.join(new_filename.as_ref()); + fs::rename(&ebuild, &new_path)?; + new_path + }; + Ok(new_path) + } + + /// Return the contents of the old PATCHES.json from Chromium OS + #[allow(dead_code)] + pub fn old_cros_patch_contents(&self, hash: &str) -> Result { + Self::old_file_contents( + hash, + &self.cros_checkout.join(CHROMIUMOS_OVERLAY_REL_PATH), + Path::new("sys-devel/llvm/files/PATCHES.json"), + ) + } + + /// Return the contents of the old PATCHES.json from android + #[allow(dead_code)] + pub fn old_android_patch_contents(&self, hash: &str) -> Result { + Self::old_file_contents( + hash, + &self.android_checkout.join(ANDROID_LLVM_REL_PATH), + Path::new("patches/PATCHES.json"), + ) + } + + /// Return the contents of an old file in git + #[allow(dead_code)] + fn old_file_contents(hash: &str, pwd: &Path, file: &Path) -> Result { + let git_ref = format!( + "{}:{}", + hash, + file.to_str() + .ok_or_else(|| anyhow!("failed to convert filepath to str"))? + ); + let output = git_cd_cmd(pwd, &["show", &git_ref])?; + if !output.status.success() { + bail!("could not get old file contents for {}", &git_ref) + } + String::from_utf8(output.stdout) + .with_context(|| format!("converting {} file contents to UTF-8", &git_ref)) + } + + /// Create the commit message + fn build_commit_msg(from: &str, to: &str, footer: &str) -> String { + format!( + "[patch_sync] Synchronize patches from {}\n\n\ + Copies new PATCHES.json changes from {} to {}\n\n{}", + from, from, to, footer + ) + } +} + +/// Return the path of an ebuild located within the given directory. +fn find_ebuild(dir: &Path) -> Result { + // TODO(ajordanr): Maybe use OnceCell for this regex? + let ebuild_matcher = Regex::new(r"(-r[0-9]+)?\.ebuild").unwrap(); + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if let Some(name) = path.file_name() { + if ebuild_matcher.is_match( + name.to_str() + .ok_or_else(|| anyhow!("converting filepath to UTF-8"))?, + ) { + return Ok(path); + } + } + } + bail!("could not find ebuild") +} + +/// Run a given git command from inside a specified git dir. +pub fn git_cd_cmd(pwd: &Path, args: I) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let output = Command::new("git").current_dir(&pwd).args(args).output()?; + if !output.status.success() { + bail!("git command failed") + } + Ok(output) +} + +pub fn repo_cd_cmd(pwd: &Path, args: I) -> Result<()> +where + I: IntoIterator, + S: AsRef, +{ + let status = Command::new("repo").current_dir(&pwd).args(args).status()?; + if !status.success() { + bail!("repo command failed") + } + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + use rand::prelude::Rng; + use std::env; + use std::fs::File; + + #[test] + fn test_revbump_ebuild() { + // Random number to append at the end of the test folder to prevent conflicts. + let rng: u32 = rand::thread_rng().gen(); + let llvm_dir = env::temp_dir().join(format!("patch_sync_test_{}", rng)); + fs::create_dir(&llvm_dir).expect("creating llvm dir in temp directory"); + + { + // With revision + let ebuild_name = "llvm-13.0_pre433403_p20211019-r10.ebuild"; + let ebuild_path = llvm_dir.join(ebuild_name); + File::create(&ebuild_path).expect("creating test ebuild file"); + let new_ebuild_path = + RepoSetupContext::rev_bump_llvm(&llvm_dir).expect("rev bumping the ebuild"); + assert!(new_ebuild_path.ends_with("llvm-13.0_pre433403_p20211019-r11.ebuild")); + fs::remove_file(new_ebuild_path).expect("removing renamed ebuild file"); + } + { + // Without revision + let ebuild_name = "llvm-13.0_pre433403_p20211019.ebuild"; + let ebuild_path = llvm_dir.join(ebuild_name); + File::create(&ebuild_path).expect("creating test ebuild file"); + let new_ebuild_path = + RepoSetupContext::rev_bump_llvm(&llvm_dir).expect("rev bumping the ebuild"); + assert!(new_ebuild_path.ends_with("llvm-13.0_pre433403_p20211019-r1.ebuild")); + fs::remove_file(new_ebuild_path).expect("removing renamed ebuild file"); + } + + fs::remove_dir(&llvm_dir).expect("removing temp test dir"); + } +} -- cgit v1.2.3 From 7fd98520753ca227c35663e39255dc313fae826c Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Tue, 14 Dec 2021 22:15:14 +0000 Subject: patch_sync: Add patch_parsing module At present, the patch_sync code does nothing as the base version only sets up the code for future reviews. This adds in the ability to parse the PATCHES.json files, check the differences across past versions, and transpose those patches to another repo. This parsing assumes the PATCHES.json now use the uniform schema, which the AOSP currently does, but CrOS may not by the time this commit lands. BUG=b:209493133 TEST=cargo build TEST=cargo test Change-Id: I8de01eae3d9555dbb000378516a5bf38e29fea8e Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3339405 Reviewed-by: George Burgess Tested-by: Jordan Abrahams Commit-Queue: Jordan Abrahams --- llvm_tools/patch_sync/src/main.rs | 51 ++++++-- llvm_tools/patch_sync/src/patch_parsing.rs | 199 +++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 llvm_tools/patch_sync/src/patch_parsing.rs diff --git a/llvm_tools/patch_sync/src/main.rs b/llvm_tools/patch_sync/src/main.rs index 8eedaaa8..514f054f 100644 --- a/llvm_tools/patch_sync/src/main.rs +++ b/llvm_tools/patch_sync/src/main.rs @@ -1,3 +1,4 @@ +mod patch_parsing; mod version_control; use anyhow::{Context, Result}; @@ -45,13 +46,43 @@ fn transpose_subcmd(args: TransposeOpt) -> Result<()> { sync_before: false, }; ctx.setup()?; - let _cros_patches_path = ctx.cros_patches_path(); - let _android_patches_path = ctx.android_patches_path(); + let cros_patches_path = ctx.cros_patches_path(); + let android_patches_path = ctx.android_patches_path(); + + // Chromium OS Patches ---------------------------------------------------- + let mut cur_cros_collection = + patch_parsing::PatchCollection::parse_from_file(&cros_patches_path) + .context("parsing cros PATCHES.json")?; + let new_cros_patches: patch_parsing::PatchCollection = { + let cros_old_patches_json = ctx.old_cros_patch_contents(&args.old_cros_ref)?; + let old_cros_collection = patch_parsing::PatchCollection::parse_from_str( + cros_patches_path.parent().unwrap().to_path_buf(), + &cros_old_patches_json, + )?; + cur_cros_collection.subtract(&old_cros_collection)? + }; + + // Android Patches ------------------------------------------------------- + let mut cur_android_collection = + patch_parsing::PatchCollection::parse_from_file(&android_patches_path) + .context("parsing android PATCHES.json")?; + let new_android_patches: patch_parsing::PatchCollection = { + let android_old_patches_json = ctx.old_android_patch_contents(&args.old_android_ref)?; + let old_android_collection = patch_parsing::PatchCollection::parse_from_str( + android_patches_path.parent().unwrap().to_path_buf(), + &android_old_patches_json, + )?; + cur_android_collection.subtract(&old_android_collection)? + }; + + // Transpose Patches ----------------------------------------------------- + new_cros_patches.transpose_write(&mut cur_cros_collection)?; + new_android_patches.transpose_write(&mut cur_android_collection)?; if !args.no_commit { return Ok(()); } - // Commit and upload for review. + // Commit and upload for review ------------------------------------------ ctx.cros_repo_upload() .context("uploading chromiumos changes")?; ctx.android_repo_upload() @@ -73,34 +104,34 @@ enum Opt { /// Transpose patches from two PATCHES.json files /// to each other. Transpose { - #[structopt(long = "cros-checkout", parse(from_os_str))] /// Path to the ChromiumOS source repo checkout. + #[structopt(long = "cros-checkout", parse(from_os_str))] cros_checkout_path: PathBuf, - #[structopt(long = "overlay-base-ref")] /// Git ref (e.g. hash) for the ChromiumOS overlay to use as the base. + #[structopt(long = "overlay-base-ref")] old_cros_ref: String, - #[structopt(long = "aosp-checkout", parse(from_os_str))] /// Path to the Android Open Source Project source repo checkout. + #[structopt(long = "aosp-checkout", parse(from_os_str))] android_checkout_path: PathBuf, - #[structopt(long = "aosp-base-ref")] /// Git ref (e.g. hash) for the llvm_android repo to use as the base. + #[structopt(long = "aosp-base-ref")] old_android_ref: String, - #[structopt(short, long)] /// Print information to stdout + #[structopt(short, long)] verbose: bool, - #[structopt(long)] /// Do not change any files. Useful in combination with `--verbose` /// Implies `--no-commit` and `--no-upload`. + #[structopt(long)] dry_run: bool, - #[structopt(long)] /// Do not commit any changes made. /// Implies `--no-upload`. + #[structopt(long)] no_commit: bool, }, } diff --git a/llvm_tools/patch_sync/src/patch_parsing.rs b/llvm_tools/patch_sync/src/patch_parsing.rs new file mode 100644 index 00000000..f95953f5 --- /dev/null +++ b/llvm_tools/patch_sync/src/patch_parsing.rs @@ -0,0 +1,199 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{copy, File}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// JSON serde struct. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PatchDictSchema { + pub rel_patch_path: String, + pub start_version: Option, + pub end_version: Option, + pub platforms: BTreeSet, + pub metadata: Option>, +} + +/// Struct to keep track of patches and their relative paths. +#[derive(Debug, Clone)] +pub struct PatchCollection { + pub patches: Vec, + pub workdir: PathBuf, +} + +impl PatchCollection { + /// Create a `PatchCollection` from a PATCHES. + pub fn parse_from_file(json_file: &Path) -> Result { + Ok(Self { + patches: serde_json::from_reader(File::open(json_file)?)?, + workdir: json_file + .parent() + .ok_or_else(|| anyhow!("failed to get json_file parent"))? + .to_path_buf(), + }) + } + + /// Create a `PatchCollection` from a string literal and a workdir. + pub fn parse_from_str(workdir: PathBuf, contents: &str) -> Result { + Ok(Self { + patches: serde_json::from_str(contents)?, + workdir, + }) + } + + #[allow(dead_code)] + /// Return true if the collection is tracking any patches. + pub fn is_empty(&self) -> bool { + self.patches.is_empty() + } + + /// Compute the set-set subtraction, returning a new `PatchCollection` which + /// keeps the minuend's wordir. + pub fn subtract(&self, subtrahend: &Self) -> Result { + let mut new_patches = Vec::new(); + // This is O(n^2) when it could be much faster, but n is always going to be less + // than 1k and speed is not important here. + for our_patch in &self.patches { + let found_in_sub = subtrahend.patches.iter().any(|sub_patch| { + let hash1 = subtrahend + .hash_from_rel_patch(sub_patch) + .expect("getting hash from subtrahend patch"); + let hash2 = self + .hash_from_rel_patch(our_patch) + .expect("getting hash from our patch"); + hash1 == hash2 + }); + if !found_in_sub { + new_patches.push(our_patch.clone()); + } + } + Ok(Self { + patches: new_patches, + workdir: self.workdir.clone(), + }) + } + + /// Copy all patches from this collection into another existing collection, and write that + /// to the existing collection's file. + pub fn transpose_write(&self, existing_collection: &mut Self) -> Result<()> { + for p in &self.patches { + let original_file_path = self.workdir.join(&p.rel_patch_path); + let copy_file_path = existing_collection.workdir.join(&p.rel_patch_path); + copy_create_parents(&original_file_path, ©_file_path)?; + existing_collection.patches.push(p.clone()); + } + existing_collection.write_patches_json("PATCHES.json") + } + + /// Write out the patch collection contents to a PATCHES.json file. + fn write_patches_json(&self, filename: &str) -> Result<()> { + let write_path = self.workdir.join(filename); + let mut new_patches_file = File::create(&write_path) + .with_context(|| format!("writing to {}", write_path.display()))?; + + let mut serialization_buffer = Vec::::new(); + // Four spaces to indent json serialization. + let mut serializer = serde_json::Serializer::with_formatter( + &mut serialization_buffer, + serde_json::ser::PrettyFormatter::with_indent(b" "), + ); + self.patches + .serialize(&mut serializer) + .with_context(|| format!("Could not serialize JSON at {}", write_path.display()))?; + // Append a newline at the end if not present. This is necessary to get + // past some pre-upload hooks. + if serialization_buffer.last() != Some(&b'\n') { + serialization_buffer.push(b'\n'); + } + new_patches_file.write_all(&serialization_buffer)?; + Ok(()) + } + + fn hash_from_rel_patch(&self, patch: &PatchDictSchema) -> Result { + hash_from_patch_path(&self.workdir.join(&patch.rel_patch_path)) + } +} + +/// Get the hash from the patch file contents. +/// +/// Not every patch file actually contains its own hash, +/// we must compute the hash ourselves when it's not found. +fn hash_from_patch(patch_contents: impl Read) -> Result { + let mut reader = BufReader::new(patch_contents); + let mut buf = String::new(); + reader.read_line(&mut buf)?; + let mut first_line_iter = buf.trim().split(' ').fuse(); + let (fst_word, snd_word) = (first_line_iter.next(), first_line_iter.next()); + if let (Some("commit" | "From"), Some(hash_str)) = (fst_word, snd_word) { + // If the first line starts with either "commit" or "From", the following + // text is almost certainly a commit hash. + Ok(hash_str.to_string()) + } else { + // This is an annoying case where the patch isn't actually a commit. + // So we'll hash the entire file, and hope that's sufficient. + let mut hasher = Sha256::new(); + hasher.update(&buf); // Have to hash the first line. + reader.read_to_string(&mut buf)?; + hasher.update(buf); // Hash the rest of the file. + let sha = hasher.finalize(); + Ok(format!("{:x}", &sha)) + } +} + +fn hash_from_patch_path(patch: &Path) -> Result { + let f = File::open(patch)?; + hash_from_patch(f) +} + +/// Copy a file from one path to another, and create any parent +/// directories along the way. +fn copy_create_parents(from: &Path, to: &Path) -> Result<()> { + let to_parent = to + .parent() + .with_context(|| format!("{} has no parent", to.display()))?; + if !to_parent.exists() { + std::fs::create_dir_all(to_parent)?; + } + + copy(&from, &to).with_context(|| { + format!( + "tried to copy file from {} to {}", + &from.display(), + &to.display() + ) + })?; + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + /// Test we can extract the hash from patch files. + #[test] + fn test_hash_from_patch() { + // Example git patch from Gerrit + let desired_hash = "004be4037e1e9c6092323c5c9268acb3ecf9176c"; + let test_file_contents = "commit 004be4037e1e9c6092323c5c9268acb3ecf9176c\n\ + Author: An Author \n\ + Date: Thu Aug 6 12:34:16 2020 -0700"; + assert_eq!( + &hash_from_patch(test_file_contents.as_bytes()).unwrap(), + desired_hash + ); + + // Example git patch from upstream + let desired_hash = "6f85225ef3791357f9b1aa097b575b0a2b0dff48"; + let test_file_contents = "From 6f85225ef3791357f9b1aa097b575b0a2b0dff48\n\ + Mon Sep 17 00:00:00 2001\n\ + From: Another Author \n\ + Date: Wed, 18 Aug 2021 15:03:03 -0700"; + assert_eq!( + &hash_from_patch(test_file_contents.as_bytes()).unwrap(), + desired_hash + ); + } +} -- cgit v1.2.3 From 0274702f8b916ec2e65b069ee3b9444e3ed6dfa8 Mon Sep 17 00:00:00 2001 From: Jordan R Abrahams Date: Thu, 23 Dec 2021 21:31:20 +0000 Subject: patch_sync: Add show subcommand This allows users to view the combined PATCHES.json, also known as the source of truth. Also makes some changes so that serializing the patches vec to a string is more natural. BUG=b:209493133 TEST=cargo test TEST=patch_sync show -s Change-Id: Ic2054c0c3c7e0bb03b698339b75e9ccc6dd68c74 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3355361 Reviewed-by: Michael Benfield Tested-by: Jordan Abrahams Commit-Queue: Jordan Abrahams --- llvm_tools/patch_sync/src/main.rs | 40 ++++++++- llvm_tools/patch_sync/src/patch_parsing.rs | 138 ++++++++++++++++++++++++++--- 2 files changed, 164 insertions(+), 14 deletions(-) diff --git a/llvm_tools/patch_sync/src/main.rs b/llvm_tools/patch_sync/src/main.rs index 514f054f..081ce01a 100644 --- a/llvm_tools/patch_sync/src/main.rs +++ b/llvm_tools/patch_sync/src/main.rs @@ -7,12 +7,17 @@ use structopt::StructOpt; fn main() -> Result<()> { match Opt::from_args() { - Opt::Show { .. } => todo!("Opt show is not implemented yet"), + Opt::Show { + cros_checkout_path, + android_checkout_path, + sync, + } => show_subcmd(cros_checkout_path, android_checkout_path, sync), Opt::Transpose { cros_checkout_path, old_cros_ref, android_checkout_path, old_android_ref, + sync, verbose, dry_run, no_commit, @@ -21,6 +26,7 @@ fn main() -> Result<()> { old_cros_ref, android_checkout_path, old_android_ref, + sync, verbose, dry_run, no_commit, @@ -28,12 +34,36 @@ fn main() -> Result<()> { } } +fn show_subcmd( + cros_checkout_path: PathBuf, + android_checkout_path: PathBuf, + sync: bool, +) -> Result<()> { + let ctx = version_control::RepoSetupContext { + cros_checkout: cros_checkout_path, + android_checkout: android_checkout_path, + sync_before: sync, + }; + ctx.setup()?; + let cros_patches_path = ctx.cros_patches_path(); + let android_patches_path = ctx.android_patches_path(); + let cur_cros_collection = patch_parsing::PatchCollection::parse_from_file(&cros_patches_path) + .context("could not parse cros PATCHES.json")?; + let cur_android_collection = + patch_parsing::PatchCollection::parse_from_file(&android_patches_path) + .context("could not parse android PATCHES.json")?; + let merged = cur_cros_collection.union(&cur_android_collection)?; + println!("{}", merged.serialize_patches()?); + Ok(()) +} + #[allow(dead_code)] struct TransposeOpt { cros_checkout_path: PathBuf, old_cros_ref: String, android_checkout_path: PathBuf, old_android_ref: String, + sync: bool, verbose: bool, dry_run: bool, no_commit: bool, @@ -43,7 +73,7 @@ fn transpose_subcmd(args: TransposeOpt) -> Result<()> { let ctx = version_control::RepoSetupContext { cros_checkout: args.cros_checkout_path, android_checkout: args.android_checkout_path, - sync_before: false, + sync_before: args.sync, }; ctx.setup()?; let cros_patches_path = ctx.cros_patches_path(); @@ -100,6 +130,8 @@ enum Opt { cros_checkout_path: PathBuf, #[structopt(parse(from_os_str))] android_checkout_path: PathBuf, + #[structopt(short, long)] + sync: bool, }, /// Transpose patches from two PATCHES.json files /// to each other. @@ -120,6 +152,10 @@ enum Opt { #[structopt(long = "aosp-base-ref")] old_android_ref: String, + /// Run repo sync before transposing. + #[structopt(short, long)] + sync: bool, + /// Print information to stdout #[structopt(short, long)] verbose: bool, diff --git a/llvm_tools/patch_sync/src/patch_parsing.rs b/llvm_tools/patch_sync/src/patch_parsing.rs index f95953f5..733451ae 100644 --- a/llvm_tools/patch_sync/src/patch_parsing.rs +++ b/llvm_tools/patch_sync/src/patch_parsing.rs @@ -39,7 +39,7 @@ impl PatchCollection { /// Create a `PatchCollection` from a string literal and a workdir. pub fn parse_from_str(workdir: PathBuf, contents: &str) -> Result { Ok(Self { - patches: serde_json::from_str(contents)?, + patches: serde_json::from_str(contents).context("parsing from str")?, workdir, }) } @@ -76,6 +76,78 @@ impl PatchCollection { }) } + pub fn union(&self, other: &Self) -> Result { + self.union_helper( + other, + |p| self.hash_from_rel_patch(p), + |p| other.hash_from_rel_patch(p), + ) + } + + fn union_helper( + &self, + other: &Self, + our_hash_f: impl Fn(&PatchDictSchema) -> Result, + their_hash_f: impl Fn(&PatchDictSchema) -> Result, + ) -> Result { + // 1. For all our patches: + // a. If there exists a matching patch hash from `other`: + // i. Create a new patch with merged platform info, + // ii. add the new patch to our new collection. + // iii. Mark the other patch as "merged" + // b. Otherwise, copy our patch to the new collection + // 2. For all unmerged patches from the `other` + // a. Copy their patch into the new collection + let mut combined_patches = Vec::new(); + let mut other_merged = vec![false; other.patches.len()]; + + // 1. + for p in &self.patches { + let our_hash = our_hash_f(p)?; + let mut found = false; + // a. + for (idx, merged) in other_merged.iter_mut().enumerate() { + if !*merged { + let other_p = &other.patches[idx]; + let their_hash = their_hash_f(other_p)?; + if our_hash == their_hash { + // i. + let new_platforms = + p.platforms.union(&other_p.platforms).cloned().collect(); + // ii. + combined_patches.push(PatchDictSchema { + rel_patch_path: p.rel_patch_path.clone(), + start_version: p.start_version, + end_version: p.end_version, + platforms: new_platforms, + metadata: p.metadata.clone(), + }); + // iii. + *merged = true; + found = true; + break; + } + } + } + // b. + if !found { + combined_patches.push(p.clone()); + } + } + // 2. + // Add any remaining, other-only patches. + for (idx, merged) in other_merged.iter().enumerate() { + if !*merged { + combined_patches.push(other.patches[idx].clone()); + } + } + + Ok(Self { + workdir: self.workdir.clone(), + patches: combined_patches, + }) + } + /// Copy all patches from this collection into another existing collection, and write that /// to the existing collection's file. pub fn transpose_write(&self, existing_collection: &mut Self) -> Result<()> { @@ -93,7 +165,11 @@ impl PatchCollection { let write_path = self.workdir.join(filename); let mut new_patches_file = File::create(&write_path) .with_context(|| format!("writing to {}", write_path.display()))?; + new_patches_file.write_all(self.serialize_patches()?.as_bytes())?; + Ok(()) + } + pub fn serialize_patches(&self) -> Result { let mut serialization_buffer = Vec::::new(); // Four spaces to indent json serialization. let mut serializer = serde_json::Serializer::with_formatter( @@ -102,14 +178,13 @@ impl PatchCollection { ); self.patches .serialize(&mut serializer) - .with_context(|| format!("Could not serialize JSON at {}", write_path.display()))?; + .context("serializing patches to JSON")?; // Append a newline at the end if not present. This is necessary to get // past some pre-upload hooks. if serialization_buffer.last() != Some(&b'\n') { serialization_buffer.push(b'\n'); } - new_patches_file.write_all(&serialization_buffer)?; - Ok(()) + Ok(std::str::from_utf8(&serialization_buffer)?.to_string()) } fn hash_from_rel_patch(&self, patch: &PatchDictSchema) -> Result { @@ -153,18 +228,13 @@ fn hash_from_patch_path(patch: &Path) -> Result { fn copy_create_parents(from: &Path, to: &Path) -> Result<()> { let to_parent = to .parent() - .with_context(|| format!("{} has no parent", to.display()))?; + .with_context(|| format!("getting parent of {}", to.display()))?; if !to_parent.exists() { std::fs::create_dir_all(to_parent)?; } - copy(&from, &to).with_context(|| { - format!( - "tried to copy file from {} to {}", - &from.display(), - &to.display() - ) - })?; + copy(&from, &to) + .with_context(|| format!("copying file from {} to {}", &from.display(), &to.display()))?; Ok(()) } @@ -196,4 +266,48 @@ mod test { desired_hash ); } + + #[test] + fn test_union() { + let patch1 = PatchDictSchema { + start_version: Some(0), + end_version: Some(1), + rel_patch_path: "a".into(), + metadata: None, + platforms: BTreeSet::from(["x".into()]), + }; + let patch2 = PatchDictSchema { + rel_patch_path: "b".into(), + platforms: BTreeSet::from(["x".into(), "y".into()]), + ..patch1.clone() + }; + let patch3 = PatchDictSchema { + platforms: BTreeSet::from(["z".into(), "x".into()]), + ..patch1.clone() + }; + let collection1 = PatchCollection { + workdir: PathBuf::new(), + patches: vec![patch1, patch2], + }; + let collection2 = PatchCollection { + workdir: PathBuf::new(), + patches: vec![patch3], + }; + let union = collection1 + .union_helper( + &collection2, + |p| Ok(p.rel_patch_path.to_string()), + |p| Ok(p.rel_patch_path.to_string()), + ) + .expect("could not create union"); + assert_eq!(union.patches.len(), 2); + assert_eq!( + union.patches[0].platforms.iter().collect::>(), + vec!["x", "z"] + ); + assert_eq!( + union.patches[1].platforms.iter().collect::>(), + vec!["x", "y"] + ); + } } -- cgit v1.2.3 From dbc8cb672a14f4a8bb136074817c71a22f743eaf Mon Sep 17 00:00:00 2001 From: Manoj Gupta Date: Wed, 5 Jan 2022 14:06:42 -0800 Subject: compiler_wrapper: Use ld.bfd to detect binutils path Use ld.bfd instead of ld to detect binutils path. This is to assist in finding packages that use GCC/Binutils ignoring portage enviroment. BUG=b:193847062 TEST=go test Change-Id: I0fd8c5c13ad2227f03ec3193c58c9bdfca4e4dc6 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3365636 Tested-by: Manoj Gupta Auto-Submit: Manoj Gupta Reviewed-by: George Burgess Commit-Queue: George Burgess --- compiler_wrapper/clang_flags.go | 2 +- compiler_wrapper/clang_flags_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go index 84149bf6..e25ed74c 100644 --- a/compiler_wrapper/clang_flags.go +++ b/compiler_wrapper/clang_flags.go @@ -126,7 +126,7 @@ func processClangFlags(builder *commandBuilder) error { // Specify the target for clang. if !builder.cfg.isHostWrapper { - linkerPath := getLinkerPath(env, builder.target.target+"-ld", builder.rootPath) + linkerPath := getLinkerPath(env, builder.target.target+"-ld.bfd", builder.rootPath) relLinkerPath, err := filepath.Rel(env.getwd(), linkerPath) if err != nil { return wrapErrorwithSourceLocf(err, "failed to make linker path %s relative to %s", diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go index 25c13c74..23aed7ef 100644 --- a/compiler_wrapper/clang_flags_test.go +++ b/compiler_wrapper/clang_flags_test.go @@ -248,7 +248,7 @@ func TestClangArchFlags(t *testing.T) { func TestClangLinkerPathProbesBinariesOnPath(t *testing.T) { withTestContext(t, func(ctx *testContext) { linkerPath := filepath.Join(ctx.tempDir, "a/b/c") - ctx.writeFile(filepath.Join(linkerPath, "x86_64-cros-linux-gnu-ld"), "") + ctx.writeFile(filepath.Join(linkerPath, "x86_64-cros-linux-gnu-ld.bfd"), "") ctx.env = []string{"PATH=nonExistantPath:" + linkerPath} cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand("./x86_64-cros-linux-gnu-clang", mainCc))) @@ -268,7 +268,7 @@ func TestClangLinkerPathEvaluatesSymlinksForBinariesOnPath(t *testing.T) { ctx.writeFile(realLinkerPath, "") firstLinkLinkerPath := filepath.Join(ctx.tempDir, "a/first/somelinker") ctx.symlink(realLinkerPath, firstLinkLinkerPath) - secondLinkLinkerPath := filepath.Join(ctx.tempDir, "a/second/x86_64-cros-linux-gnu-ld") + secondLinkLinkerPath := filepath.Join(ctx.tempDir, "a/second/x86_64-cros-linux-gnu-ld.bfd") ctx.symlink(firstLinkLinkerPath, secondLinkLinkerPath) ctx.env = []string{"PATH=nonExistantPath:" + filepath.Dir(secondLinkLinkerPath)} -- cgit v1.2.3