From 27fee01174f8448b8a73ba6352ee24589cc92bd3 Mon Sep 17 00:00:00 2001 From: chrome-bot Date: Mon, 13 Jan 2020 16:50:47 -0800 Subject: afdo_metadata: Publish new profiles for kernel 3.18. Update chromeos-kernel-3_18 from R81-12739.12-1578524538 to R81-12814.0-1578912053 Change-Id: Icfdc2be38124baa29c319e6bcaa94cbd2e9a7c31 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1998600 --- afdo_metadata/kernel_afdo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index bf869c93..7bc27de8 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -6,6 +6,6 @@ "name": "R81-12739.12-1578524221" }, "chromeos-kernel-3_18": { - "name": "R81-12739.12-1578524538" + "name": "R81-12814.0-1578912053" } } \ No newline at end of file -- cgit v1.2.3 From c977ae00dfd11ebd7c9212b04d0cf1808ea9bfc6 Mon Sep 17 00:00:00 2001 From: chrome-bot Date: Mon, 13 Jan 2020 22:43:43 -0800 Subject: afdo_metadata: Publish new profiles for kernel 4.4. Update chromeos-kernel-4_4 from R81-12739.12-1578524221 to R81-12814.0-1578911615 Change-Id: Idf3377bef43b5e6bbb1e0cecdee6fa86efc4131b Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1999556 --- afdo_metadata/kernel_afdo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 7bc27de8..350ac031 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -3,7 +3,7 @@ "name": "R81-12607.58-1578524353" }, "chromeos-kernel-4_4": { - "name": "R81-12739.12-1578524221" + "name": "R81-12814.0-1578911615" }, "chromeos-kernel-3_18": { "name": "R81-12814.0-1578912053" -- cgit v1.2.3 From 0410b6515691a2fd3410917849d32495490d6999 Mon Sep 17 00:00:00 2001 From: Zhizhou Yang Date: Tue, 14 Jan 2020 16:49:12 -0800 Subject: crosperf: fix pylint for crosperf This is the first step of python 3 migration for crosperf TEST=Passed presubmit for all .py files in crosperf BUG=chromium:1011676 Change-Id: I615d0323c81ba688c8ce2aa0cac307035d0b882b Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2001696 Commit-Queue: Zhizhou Yang Tested-by: Zhizhou Yang Auto-Submit: Zhizhou Yang Reviewed-by: George Burgess --- crosperf/column_chart.py | 8 +++- crosperf/compare_machines.py | 2 + crosperf/config.py | 2 + crosperf/config_unittest.py | 15 ++++--- crosperf/download_images_buildid_test.py | 21 ++++++---- crosperf/download_images_unittest.py | 3 +- crosperf/experiment_status.py | 26 +++++++----- crosperf/field.py | 6 ++- crosperf/flag_test_unittest.py | 14 ++++--- crosperf/generate_report.py | 11 +++-- crosperf/generate_report_unittest.py | 44 ++++++++++++-------- crosperf/help.py | 6 ++- crosperf/image_checksummer.py | 6 ++- crosperf/machine_image_manager.py | 15 +++++-- crosperf/machine_image_manager_unittest.py | 66 +++++++++++++++++------------- crosperf/machine_manager.py | 24 ++++++----- crosperf/results_report.py | 4 +- crosperf/results_report_templates.py | 2 + crosperf/results_report_unittest.py | 22 +++++----- crosperf/settings.py | 2 +- crosperf/settings_unittest.py | 4 +- crosperf/test_flag.py | 6 ++- crosperf/translate_xbuddy.py | 7 +++- 23 files changed, 196 insertions(+), 120 deletions(-) diff --git a/crosperf/column_chart.py b/crosperf/column_chart.py index 7e6821d0..400979ee 100644 --- a/crosperf/column_chart.py +++ b/crosperf/column_chart.py @@ -1,4 +1,8 @@ -# Copyright 2011 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Module to draw column chart.""" @@ -7,7 +11,7 @@ class ColumnChart(object): def __init__(self, title, width, height): self.title = title - self.chart_div = filter(str.isalnum, title) + self.chart_div = ''.join(t for t in title if t.isalnum()) self.width = width self.height = height self.columns = [] diff --git a/crosperf/compare_machines.py b/crosperf/compare_machines.py index 34513a87..c73f8756 100644 --- a/crosperf/compare_machines.py +++ b/crosperf/compare_machines.py @@ -1,6 +1,8 @@ +# -*- coding: utf-8 -*- # Copyright 2014 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Module to compare two machines.""" from __future__ import print_function diff --git a/crosperf/config.py b/crosperf/config.py index 76175660..61ad9c1a 100644 --- a/crosperf/config.py +++ b/crosperf/config.py @@ -1,6 +1,8 @@ +# -*- coding: utf-8 -*- # Copyright 2011 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """A configure file.""" config = {} diff --git a/crosperf/config_unittest.py b/crosperf/config_unittest.py index 637dae9e..9e71539d 100755 --- a/crosperf/config_unittest.py +++ b/crosperf/config_unittest.py @@ -1,14 +1,17 @@ #!/usr/bin/env python2 -# -# Copyright 2014 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2014 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Unit tests for config.py""" from __future__ import print_function -import config - import unittest +import config + class ConfigTestCase(unittest.TestCase): """Class for the config unit tests.""" @@ -16,7 +19,7 @@ class ConfigTestCase(unittest.TestCase): def test_config(self): # Verify that config exists, that it's a dictionary, and that it's # empty. - self.assertTrue(type(config.config) is dict) + self.assertTrue(isinstance(config.config, dict)) self.assertEqual(len(config.config), 0) # Verify that attempting to get a non-existant key out of the @@ -42,7 +45,7 @@ class ConfigTestCase(unittest.TestCase): # Verify that config exists, that it's a dictionary, and that it's # empty. - self.assertTrue(type(config.config) is dict) + self.assertTrue(isinstance(config.config, dict)) self.assertEqual(len(config.config), 0) diff --git a/crosperf/download_images_buildid_test.py b/crosperf/download_images_buildid_test.py index 3e7f00c1..a376691c 100755 --- a/crosperf/download_images_buildid_test.py +++ b/crosperf/download_images_buildid_test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python2 -# -# Copyright 2014 Google Inc. All Rights Reserved +# -*- coding: utf-8 -*- +# Copyright 2014 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Test translation of xbuddy names.""" from __future__ import print_function @@ -10,12 +13,12 @@ import sys import download_images -#On May 1, 2014: -#latest : lumpy-release/R34-5500.132.0 -#latest-beta : lumpy-release/R35-5712.43.0 -#latest-official: lumpy-release/R36-5814.0.0 -#latest-dev : lumpy-release/R36-5814.0.0 -#latest-canary : lumpy-release/R36-5814.0.0 +# On May 1, 2014: +# latest : lumpy-release/R34-5500.132.0 +# latest-beta : lumpy-release/R35-5712.43.0 +# latest-official: lumpy-release/R36-5814.0.0 +# latest-dev : lumpy-release/R36-5814.0.0 +# latest-canary : lumpy-release/R36-5814.0.0 class ImageDownloaderBuildIDTest(object): @@ -55,7 +58,7 @@ class ImageDownloaderBuildIDTest(object): sys.exit(1) def assertIsNotNone(self, arg, arg_name): - if arg == None: + if arg is None: self.tests_failed = self.tests_failed + 1 self.assert_failure('%s is not None' % arg_name) diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py index dd83c3fa..94504d3f 100755 --- a/crosperf/download_images_unittest.py +++ b/crosperf/download_images_unittest.py @@ -9,9 +9,10 @@ from __future__ import print_function import os -import mock import unittest +import mock + import download_images from cros_utils import command_executer from cros_utils import logger diff --git a/crosperf/experiment_status.py b/crosperf/experiment_status.py index c6610433..2ac47c74 100644 --- a/crosperf/experiment_status.py +++ b/crosperf/experiment_status.py @@ -1,8 +1,11 @@ +# -*- coding: utf-8 -*- # Copyright 2011 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """The class to show the banner.""" +from __future__ import division from __future__ import print_function import collections @@ -25,7 +28,7 @@ class ExperimentStatus(object): bar_length = 50 done_char = '>' undone_char = ' ' - num_complete_chars = bar_length * num_complete / num_total + num_complete_chars = bar_length * num_complete // num_total num_undone_chars = bar_length - num_complete_chars ret += ' [%s%s]' % (num_complete_chars * done_char, num_undone_chars * undone_char) @@ -42,8 +45,8 @@ class ExperimentStatus(object): if self.completed != self.experiment.num_complete: self.completed = self.experiment.num_complete self.new_job_start_time = current_time - time_completed_jobs = (elapsed_time - - (current_time - self.new_job_start_time)) + time_completed_jobs = ( + elapsed_time - (current_time - self.new_job_start_time)) # eta is calculated as: # ETA = (num_jobs_not_yet_started * estimated_time_per_job) # + time_left_for_current_job @@ -64,10 +67,11 @@ class ExperimentStatus(object): # first long job, after a series of short jobs). For now, if that # happens, we set the ETA to "Unknown." # - eta_seconds = (float(self.num_total - self.experiment.num_complete - 1) * - time_completed_jobs / self.experiment.num_run_complete + - (time_completed_jobs / self.experiment.num_run_complete - - (current_time - self.new_job_start_time))) + eta_seconds = ( + float(self.num_total - self.experiment.num_complete - 1) * + time_completed_jobs / self.experiment.num_run_complete + + (time_completed_jobs / self.experiment.num_run_complete - + (current_time - self.new_job_start_time))) eta_seconds = int(eta_seconds) if eta_seconds > 0: @@ -91,7 +95,7 @@ class ExperimentStatus(object): status_bins[benchmark_run.timeline.GetLastEvent()].append(benchmark_run) status_strings = [] - for key, val in status_bins.iteritems(): + for key, val in status_bins.items(): if key == 'RUNNING': get_description = self._GetNamesAndIterations else: @@ -129,16 +133,16 @@ class ExperimentStatus(object): grouped_benchmarks[benchmark_run.label.name].append(benchmark_run) output_segs = [] - for label_name, label_runs in grouped_benchmarks.iteritems(): + for label_name, label_runs in grouped_benchmarks.items(): strings = [] benchmark_iterations = collections.defaultdict(list) for benchmark_run in label_runs: assert benchmark_run.label.name == label_name benchmark_name = benchmark_run.benchmark.name benchmark_iterations[benchmark_name].append(benchmark_run.iteration) - for key, val in benchmark_iterations.iteritems(): + for key, val in benchmark_iterations.items(): val.sort() - iterations = ','.join(map(str, val)) + iterations = ','.join(str(v) for v in val) strings.append('{} [{}]'.format(key, iterations)) output_segs.append(' ' + label_name + ': ' + ', '.join(strings) + '\n') diff --git a/crosperf/field.py b/crosperf/field.py index 6821d4d3..f6300f9f 100644 --- a/crosperf/field.py +++ b/crosperf/field.py @@ -1,4 +1,8 @@ -# Copyright 2011 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Module to represent a Field in an experiment file.""" diff --git a/crosperf/flag_test_unittest.py b/crosperf/flag_test_unittest.py index 0e743274..cb0e59e6 100755 --- a/crosperf/flag_test_unittest.py +++ b/crosperf/flag_test_unittest.py @@ -1,13 +1,17 @@ #!/usr/bin/env python2 -# -# Copyright 2014 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2014 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """The unittest of flags.""" from __future__ import print_function -import test_flag import unittest +import test_flag + class FlagTestCase(unittest.TestCase): """The unittest class.""" @@ -15,7 +19,7 @@ class FlagTestCase(unittest.TestCase): def test_test_flag(self): # Verify that test_flag.is_test exists, that it is a list, # and that it contains 1 element. - self.assertTrue(type(test_flag.is_test) is list) + self.assertTrue(isinstance(test_flag.is_test, list)) self.assertEqual(len(test_flag.is_test), 1) # Verify that the getting the flag works and that the flag @@ -33,7 +37,7 @@ class FlagTestCase(unittest.TestCase): # Verify that test_flag.is_test still exists, that it still is a # list, and that it still contains 1 element. - self.assertTrue(type(test_flag.is_test) is list) + self.assertTrue(isinstance(test_flag.is_test, list)) self.assertEqual(len(test_flag.is_test), 1) diff --git a/crosperf/generate_report.py b/crosperf/generate_report.py index fd7a2cf7..cea95a2e 100755 --- a/crosperf/generate_report.py +++ b/crosperf/generate_report.py @@ -1,8 +1,9 @@ #!/usr/bin/env python2 -# +# -*- coding: utf-8 -*- # Copyright 2016 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Given a specially-formatted JSON object, generates results report(s). The JSON object should look like: @@ -67,8 +68,7 @@ def CountBenchmarks(benchmark_runs): def _MaxLen(results): return 0 if not results else max(len(r) for r in results) - return [(name, _MaxLen(results)) - for name, results in benchmark_runs.iteritems()] + return [(name, _MaxLen(results)) for name, results in benchmark_runs.items()] def CutResultsInPlace(results, max_keys=50, complain_on_update=True): @@ -130,9 +130,14 @@ def CutResultsInPlace(results, max_keys=50, complain_on_update=True): def _ConvertToASCII(obj): """Convert an object loaded from JSON to ASCII; JSON gives us unicode.""" + # In python 3, we can directly return the unicode loaded from json. + if sys.version_info.major == 3: + return obj + # TODO(zhizhouy): Drop the following code once migrated to python 3. # Using something like `object_hook` is insufficient, since it only fires on # actual JSON objects. `encoding` fails, too, since the default decoder always # uses unicode() to decode strings. + # pylint: disable=unicode-builtin if isinstance(obj, unicode): return str(obj) if isinstance(obj, dict): diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py index bbb0c0ae..465db29b 100755 --- a/crosperf/generate_report_unittest.py +++ b/crosperf/generate_report_unittest.py @@ -1,23 +1,28 @@ #!/usr/bin/env python2 -# +# -*- coding: utf-8 -*- # Copyright 2016 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Test for generate_report.py.""" from __future__ import division from __future__ import print_function -from StringIO import StringIO - import copy import json -import mock -import test_flag import unittest +import mock import generate_report import results_report +import test_flag + +# pylint: disable=deprecated-module +try: + from StringIO import StringIO # for Python 2 +except ImportError: + from io import StringIO # for Python 3 class _ContextualStringIO(StringIO): @@ -44,7 +49,7 @@ class GenerateReportTests(unittest.TestCase): } results = generate_report.CountBenchmarks(runs) expected_results = [('foo', 4), ('bar', 0), ('baz', 3)] - self.assertItemsEqual(expected_results, results) + self.assertEqual(sorted(expected_results), sorted(results)) def testCutResultsInPlace(self): bench_data = { @@ -77,10 +82,11 @@ class GenerateReportTests(unittest.TestCase): bench_data, max_keys=max_keys, complain_on_update=False) # Cuts should be in-place. self.assertIs(results, bench_data) - self.assertItemsEqual(original_bench_data.keys(), bench_data.keys()) - for bench_name, original_runs in original_bench_data.iteritems(): + self.assertEqual( + sorted(original_bench_data.keys()), sorted(bench_data.keys())) + for bench_name, original_runs in original_bench_data.items(): bench_runs = bench_data[bench_name] - self.assertEquals(len(original_runs), len(bench_runs)) + self.assertEqual(len(original_runs), len(bench_runs)) # Order of these sub-lists shouldn't have changed. for original_list, new_list in zip(original_runs, bench_runs): self.assertEqual(len(original_list), len(new_list)) @@ -106,9 +112,9 @@ class GenerateReportTests(unittest.TestCase): # Just reach into results assuming we know it otherwise outputs things # sanely. If it doesn't, testCutResultsInPlace should give an indication as # to what, exactly, is broken. - self.assertEqual(results['foo'][0][0].items(), [('retval', 0)]) - self.assertEqual(results['bar'][0][0].items(), [('retval', 1)]) - self.assertEqual(results['baz'][0][0].items(), []) + self.assertEqual(list(results['foo'][0][0].items()), [('retval', 0)]) + self.assertEqual(list(results['bar'][0][0].items()), [('retval', 1)]) + self.assertEqual(list(results['baz'][0][0].items()), []) def _RunMainWithInput(self, args, input_obj): assert '-i' not in args @@ -129,11 +135,13 @@ class GenerateReportTests(unittest.TestCase): self.assertEqual(0, return_code) self.assertEqual(mock_run_actions.call_count, 1) ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] - self.assertItemsEqual(ctors, [ - results_report.JSONResultsReport, - results_report.TextResultsReport, - results_report.HTMLResultsReport, - ]) + self.assertEqual( + sorted(ctors), + sorted([ + results_report.JSONResultsReport, + results_report.TextResultsReport, + results_report.HTMLResultsReport, + ])) @mock.patch('generate_report.RunActions') def testMainSelectsHTMLIfNoReportsGiven(self, mock_run_actions): @@ -142,7 +150,7 @@ class GenerateReportTests(unittest.TestCase): self.assertEqual(0, return_code) self.assertEqual(mock_run_actions.call_count, 1) ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] - self.assertItemsEqual(ctors, [results_report.HTMLResultsReport]) + self.assertEqual(ctors, [results_report.HTMLResultsReport]) # We only mock print_exc so we don't have exception info printed to stdout. @mock.patch('generate_report.WriteFile', side_effect=ValueError('Oh noo')) diff --git a/crosperf/help.py b/crosperf/help.py index 61ed8ea2..4409b770 100644 --- a/crosperf/help.py +++ b/crosperf/help.py @@ -1,4 +1,8 @@ -# Copyright 2011 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Module to print help message.""" from __future__ import print_function diff --git a/crosperf/image_checksummer.py b/crosperf/image_checksummer.py index f5862e4d..8ac5be25 100644 --- a/crosperf/image_checksummer.py +++ b/crosperf/image_checksummer.py @@ -1,4 +1,8 @@ -# Copyright 2011 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Compute image checksum.""" from __future__ import print_function diff --git a/crosperf/machine_image_manager.py b/crosperf/machine_image_manager.py index 2ad750d3..ffdd6436 100644 --- a/crosperf/machine_image_manager.py +++ b/crosperf/machine_image_manager.py @@ -1,8 +1,14 @@ +# -*- coding: utf-8 -*- # Copyright 2015 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """MachineImageManager allocates images to duts.""" +from __future__ import print_function + +import functools + class MachineImageManager(object): """Management of allocating images to duts. @@ -143,8 +149,9 @@ class MachineImageManager(object): self.dut_name_ordinal_[dut.name] = idx # Generate initial matrix containg 'X' or ' '. - self.matrix_ = [['X' if (l.remote and len(l.remote)) else ' ' \ - for _ in range(self.n_duts_)] for l in self.labels_] + self.matrix_ = [['X' if l.remote else ' ' + for _ in range(self.n_duts_)] + for l in self.labels_] for ol, l in enumerate(self.labels_): if l.remote: for r in l.remote: @@ -291,8 +298,8 @@ class MachineImageManager(object): if v == ' ': # Before we put a 'Y', we check how many Y column 'j' has. # Note y[0] is row idx, y[1] is the cell value. - ny = reduce(lambda x, y: x + 1 if (y[1] == 'Y') else x, - self.matrix_vertical_generator(j), 0) + ny = functools.reduce(lambda x, y: x + 1 if (y[1] == 'Y') else x, + self.matrix_vertical_generator(j), 0) if ny < N: self.matrix_[level][j] = 'Y' if self._compute_initial_allocation_internal(level + 1, N): diff --git a/crosperf/machine_image_manager_unittest.py b/crosperf/machine_image_manager_unittest.py index 02afaa06..062c185f 100755 --- a/crosperf/machine_image_manager_unittest.py +++ b/crosperf/machine_image_manager_unittest.py @@ -1,6 +1,9 @@ #!/usr/bin/env python2 +# -*- coding: utf-8 -*- +# Copyright 2015 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. -# Copyright 2015 Google Inc. All Rights Reserved. """Unit tests for the MachineImageManager class.""" from __future__ import print_function @@ -98,62 +101,67 @@ class MachineImageManagerTester(unittest.TestCase): def test_case1(self): labels = [ - MockLabel('l1', ['m1', 'm2']), MockLabel('l2', ['m2', 'm3']), MockLabel( - 'l3', ['m1']) + MockLabel('l1', ['m1', 'm2']), + MockLabel('l2', ['m2', 'm3']), + MockLabel('l3', ['m1']) ] duts = [MockDut('m1'), MockDut('m2'), MockDut('m3')] mim = MachineImageManager(labels, duts) - self.assertTrue(mim.matrix_ == [[' ', ' ', 'X'], ['X', ' ', ' '], - [' ', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [[' ', ' ', 'X'], ['X', ' ', ' '], [' ', 'X', 'X']]) mim.compute_initial_allocation() - self.assertTrue(mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], - ['Y', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], ['Y', 'X', 'X']]) def test_case2(self): labels = [ - MockLabel('l1', ['m1', 'm2']), MockLabel('l2', ['m2', 'm3']), MockLabel( - 'l3', ['m1']) + MockLabel('l1', ['m1', 'm2']), + MockLabel('l2', ['m2', 'm3']), + MockLabel('l3', ['m1']) ] duts = [MockDut('m1'), MockDut('m2'), MockDut('m3')] mim = MachineImageManager(labels, duts) - self.assertTrue(mim.matrix_ == [[' ', ' ', 'X'], ['X', ' ', ' '], - [' ', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [[' ', ' ', 'X'], ['X', ' ', ' '], [' ', 'X', 'X']]) mim.compute_initial_allocation() - self.assertTrue(mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], - ['Y', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], ['Y', 'X', 'X']]) def test_case3(self): labels = [ - MockLabel('l1', ['m1', 'm2']), MockLabel('l2', ['m2', 'm3']), MockLabel( - 'l3', ['m1']) + MockLabel('l1', ['m1', 'm2']), + MockLabel('l2', ['m2', 'm3']), + MockLabel('l3', ['m1']) ] duts = [MockDut('m1', labels[0]), MockDut('m2'), MockDut('m3')] mim = MachineImageManager(labels, duts) mim.compute_initial_allocation() - self.assertTrue(mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], - ['Y', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], ['Y', 'X', 'X']]) def test_case4(self): labels = [ - MockLabel('l1', ['m1', 'm2']), MockLabel('l2', ['m2', 'm3']), MockLabel( - 'l3', ['m1']) + MockLabel('l1', ['m1', 'm2']), + MockLabel('l2', ['m2', 'm3']), + MockLabel('l3', ['m1']) ] duts = [MockDut('m1'), MockDut('m2', labels[0]), MockDut('m3')] mim = MachineImageManager(labels, duts) mim.compute_initial_allocation() - self.assertTrue(mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], - ['Y', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [[' ', 'Y', 'X'], ['X', ' ', 'Y'], ['Y', 'X', 'X']]) def test_case5(self): labels = [ - MockLabel('l1', ['m3']), MockLabel('l2', ['m3']), MockLabel( - 'l3', ['m1']) + MockLabel('l1', ['m3']), + MockLabel('l2', ['m3']), + MockLabel('l3', ['m1']) ] duts = self.gen_duts_by_name('m1', 'm2', 'm3') mim = MachineImageManager(labels, duts) self.assertTrue(mim.compute_initial_allocation()) - self.assertTrue(mim.matrix_ == [['X', 'X', 'Y'], ['X', 'X', 'Y'], - ['Y', 'X', 'X']]) + self.assertTrue( + mim.matrix_ == [['X', 'X', 'Y'], ['X', 'X', 'Y'], ['Y', 'X', 'X']]) def test_2x2_with_allocation(self): labels = [MockLabel('l0'), MockLabel('l1')] @@ -249,13 +257,13 @@ class MachineImageManagerTester(unittest.TestCase): self.assertTrue(mim.allocate(mim.duts_[3]) is None) self.assertTrue(mim.allocate(mim.duts_[2]) is None) self.assertTrue(mim.allocate(mim.duts_[1]) == mim.labels_[1]) - self.assertTrue(mim.allocate(mim.duts_[1]) == None) - self.assertTrue(mim.allocate(mim.duts_[0]) == None) + self.assertTrue(mim.allocate(mim.duts_[1]) is None) + self.assertTrue(mim.allocate(mim.duts_[0]) is None) self.assertTrue(mim.label_duts_[0] == [2, 3]) self.assertTrue(mim.label_duts_[1] == [0, 3, 1]) self.assertTrue(mim.label_duts_[2] == [3, 1]) - self.assertTrue(mim.allocate_log_ == [(0, 2), (2, 3), (1, 0), (2, 1), - (1, 3), (0, 3), (1, 1)]) + self.assertListEqual(mim.allocate_log_, [(0, 2), (2, 3), (1, 0), (2, 1), + (1, 3), (0, 3), (1, 1)]) def test_cornercase_1(self): """This corner case is brought up by Caroline. diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index 2cdaca3f..6ed7e513 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -1,15 +1,15 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Machine Manager module.""" +from __future__ import division from __future__ import print_function import collections -import file_lock_machine import hashlib -import image_chromeos import math import os.path import re @@ -17,6 +17,8 @@ import sys import threading import time +import file_lock_machine +import image_chromeos import test_flag from cros_utils import command_executer from cros_utils import logger @@ -124,8 +126,8 @@ class CrosMachine(object): self.phys_kbytes = phys_kbytes def _GetMemoryInfo(self): - #TODO yunlian: when the machine in rebooting, it will not return - #meminfo, the assert does not catch it either + # TODO yunlian: when the machine in rebooting, it will not return + # meminfo, the assert does not catch it either command = 'cat /proc/meminfo' ret, self.meminfo, _ = self.ce.CrosRunCommandWOutput( command, machine=self.name, chromeos_root=self.chromeos_root) @@ -159,7 +161,7 @@ class CrosMachine(object): command, machine=self.name, chromeos_root=self.chromeos_root) b = if_out.splitlines() a = [l for l in b if 'Product' in l] - if len(a): + if a: self.machine_id = a[0] return command = 'ifconfig' @@ -167,11 +169,11 @@ class CrosMachine(object): command, machine=self.name, chromeos_root=self.chromeos_root) b = if_out.splitlines() a = [l for l in b if 'HWaddr' in l] - if len(a): + if a: self.machine_id = '_'.join(a) return a = [l for l in b if 'ether' in l] - if len(a): + if a: self.machine_id = '_'.join(a) return assert 0, 'Could not get machine_id from machine: %s' % self.name @@ -515,7 +517,7 @@ class MachineManager(object): dic[machine.cpuinfo].append(label.name) break output_segs = [] - for key, v in dic.iteritems(): + for key, v in dic.items(): output = ' '.join(v) output += '\n-------------------\n' output += key @@ -630,7 +632,7 @@ power management: self.test_run = None self.chromeos_root = chromeos_root self.checksum_string = re.sub(r'\d', '', name) - #In test, we assume "lumpy1", "lumpy2" are the same machine. + # In test, we assume "lumpy1", "lumpy2" are the same machine. self.machine_checksum = self._GetMD5Checksum(self.checksum_string) self.log_level = log_level self.label = None @@ -684,8 +686,8 @@ class MockMachineManager(MachineManager): return machine return None - def ImageMachine(self, machine_name, label): - if machine_name or label: + def ImageMachine(self, machine, label): + if machine or label: return 0 return 1 diff --git a/crosperf/results_report.py b/crosperf/results_report.py index 22719027..81591db7 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -375,8 +375,8 @@ class TextResultsReport(ResultsReport): def GetTotalWaitCooldownTime(self): """Get cooldown wait time in seconds from experiment benchmark runs. - Returns: - Dictionary {'dut': int(wait_time_in_seconds)} + Returns: + Dictionary {'dut': int(wait_time_in_seconds)} """ waittime_dict = {} for dut in self.experiment.machine_manager.GetMachines(): diff --git a/crosperf/results_report_templates.py b/crosperf/results_report_templates.py index 15ce5827..3c5258c9 100644 --- a/crosperf/results_report_templates.py +++ b/crosperf/results_report_templates.py @@ -1,6 +1,8 @@ +# -*- coding: utf-8 -*- # Copyright 2016 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Text templates used by various parts of results_report.""" from __future__ import print_function diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py index 48fa0c38..dfcce725 100755 --- a/crosperf/results_report_unittest.py +++ b/crosperf/results_report_unittest.py @@ -130,19 +130,19 @@ def _InjectSuccesses(experiment, how_many, keyvals, for_benchmark=0): machine_manager.AddMachine('testing_machine') machine = next( m for m in machine_manager.GetMachines() if m.name == 'testing_machine') - for label in experiment.labels: - def MakeSuccessfulRun(n): - run = MockBenchmarkRun('mock_success%d' % (n,), bench, label, - 1 + n + num_runs, cache_conditions, - machine_manager, log, log_level, share_cache, {}) - mock_result = MockResult(log, label, log_level, machine) - mock_result.keyvals = keyvals - run.result = mock_result - return run + def MakeSuccessfulRun(n, label): + run = MockBenchmarkRun('mock_success%d' % (n,), bench, label, + 1 + n + num_runs, cache_conditions, machine_manager, + log, log_level, share_cache, {}) + mock_result = MockResult(log, label, log_level, machine) + mock_result.keyvals = keyvals + run.result = mock_result + return run + for label in experiment.labels: experiment.benchmark_runs.extend( - MakeSuccessfulRun(n) for n in range(how_many)) + MakeSuccessfulRun(n, label) for n in range(how_many)) return experiment @@ -429,7 +429,7 @@ class PerfReportParserTest(unittest.TestCase): def testParserParsesRealWorldPerfReport(self): report = ParseStandardPerfReport(self._ReadRealPerfReport()) - self.assertItemsEqual(['cycles', 'instructions'], report.keys()) + self.assertItemsEqual(['cycles', 'instructions'], sorted(report.keys())) # Arbitrarily selected known percentages from the perf report. known_cycles_percentages = { diff --git a/crosperf/settings.py b/crosperf/settings.py index 708697b1..41530d97 100644 --- a/crosperf/settings.py +++ b/crosperf/settings.py @@ -1,4 +1,4 @@ -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/crosperf/settings_unittest.py b/crosperf/settings_unittest.py index 8140e9b9..e6126923 100755 --- a/crosperf/settings_unittest.py +++ b/crosperf/settings_unittest.py @@ -37,7 +37,7 @@ class TestSettings(unittest.TestCase): settings_parent = {'fake_parent_entry': 0} self.settings.SetParentSettings(settings_parent) self.assertIsNotNone(self.settings.parent) - self.assertEqual(type(self.settings.parent), dict) + self.assertTrue(isinstance(self.settings.parent, dict)) self.assertEqual(self.settings.parent, settings_parent) def test_add_field(self): @@ -88,7 +88,7 @@ class TestSettings(unittest.TestCase): description="A comma-separated list of ip's of " 'chromeos devices to run ' 'experiments on.')) - self.assertEqual(type(self.settings.fields), dict) + self.assertTrue(isinstance(self.settings.fields, dict)) self.assertEqual(len(self.settings.fields), 2) res = self.settings.fields['remote'] self.assertEqual(res.Get(), []) diff --git a/crosperf/test_flag.py b/crosperf/test_flag.py index 70918693..6fa3b589 100644 --- a/crosperf/test_flag.py +++ b/crosperf/test_flag.py @@ -1,4 +1,8 @@ -# Copyright 2011 Google Inc. All Rights Reserved. +# -*- coding: utf-8 -*- +# Copyright 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """A global variable for testing.""" is_test = [False] diff --git a/crosperf/translate_xbuddy.py b/crosperf/translate_xbuddy.py index 20fcc704..ca7058ea 100644 --- a/crosperf/translate_xbuddy.py +++ b/crosperf/translate_xbuddy.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Module to translate the xbuddy config.""" from __future__ import print_function @@ -14,7 +19,7 @@ else: ' and try again.') sys.exit(0) -#pylint: disable=import-error +# pylint: disable=import-error,wrong-import-position import xbuddy -- cgit v1.2.3 From b47813efda60622787afcb0a9bc3070031ac4e9f Mon Sep 17 00:00:00 2001 From: Jian Cai Date: Thu, 16 Jan 2020 11:12:54 -0800 Subject: patch_manager: appease pylint Fix unused-argument complaints in unit tests. BUG=chromium:1042953 TEST=verified locally. Change-Id: I7037fc10f8160a5f1f8929a3256e91255dd125ac Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2006330 Reviewed-by: George Burgess Reviewed-by: Manoj Gupta Tested-by: Jian Cai --- llvm_tools/patch_manager.py | 14 ++++++-------- llvm_tools/patch_manager_unittest.py | 10 +++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/llvm_tools/patch_manager.py b/llvm_tools/patch_manager.py index 910ecfc9..82a73ed8 100755 --- a/llvm_tools/patch_manager.py +++ b/llvm_tools/patch_manager.py @@ -8,17 +8,17 @@ from __future__ import print_function -from collections import namedtuple -from failure_modes import FailureModes -from subprocess_helpers import check_call -from subprocess_helpers import check_output - import argparse import json -import get_llvm_hash import os import subprocess import sys +from collections import namedtuple + +import get_llvm_hash +from failure_modes import FailureModes +from subprocess_helpers import check_call +from subprocess_helpers import check_output def is_directory(dir_path): @@ -279,8 +279,6 @@ def UpdatePatchMetadataFile(patch_metadata_file, patches): def GetCommitHashesForBisection(src_path, good_svn_version, bad_svn_version): """Gets the good and bad commit hashes required by `git bisect start`.""" - new_llvm_hash = LLVMHash() - bad_commit_hash = get_llvm_hash.GetGitHashFrom(src_path, bad_svn_version) good_commit_hash = get_llvm_hash.GetGitHashFrom(src_path, good_svn_version) diff --git a/llvm_tools/patch_manager_unittest.py b/llvm_tools/patch_manager_unittest.py index 90200ab8..688c2fe6 100755 --- a/llvm_tools/patch_manager_unittest.py +++ b/llvm_tools/patch_manager_unittest.py @@ -14,11 +14,11 @@ import subprocess import unittest import unittest.mock as mock +import patch_manager from failure_modes import FailureModes from test_helpers import CallCountsToMockFunctions from test_helpers import CreateTemporaryJsonFile from test_helpers import WritePrettyJsonFile -import patch_manager class PatchManagerTest(unittest.TestCase): @@ -326,8 +326,8 @@ class PatchManagerTest(unittest.TestCase): # Simulate behavior of 'GetPathToPatch()' when the absolute path to the # patch does not exist. def PathToPatchDoesNotExist(filesdir_path, rel_patch_path): - raise ValueError( - 'The absolute path to %s does not exist' % abs_patch_path) + raise ValueError('The absolute path to %s does not exist' % os.path.join( + filesdir_path, rel_patch_path)) # Use the test function to simulate the behavior of 'GetPathToPatch()'. mock_get_path_to_patch.side_effect = PathToPatchDoesNotExist @@ -463,7 +463,7 @@ class PatchManagerTest(unittest.TestCase): # Simulate behavior for 'ApplyPatch()' when applying multiple applicable # patches. @CallCountsToMockFunctions - def MultipleCallsToApplyPatches(call_count, src_path, path_to_patch): + def MultipleCallsToApplyPatches(call_count, _, path_to_patch): if call_count < 3: self.assertEqual( path_to_patch, @@ -569,7 +569,7 @@ class PatchManagerTest(unittest.TestCase): # Simulate behavior for 'ApplyPatch()' when applying multiple applicable # patches. @CallCountsToMockFunctions - def MultipleCallsToApplyPatches(call_count, src_path, path_to_patch): + def MultipleCallsToApplyPatches(call_count, _, path_to_patch): if call_count < 3: self.assertEqual( path_to_patch, -- cgit v1.2.3 From 197e3714bdcd3ffd3a9faa7568293727e0cd1fb9 Mon Sep 17 00:00:00 2001 From: Jian Cai Date: Thu, 16 Jan 2020 17:55:34 -0800 Subject: patch_manager_unittest: rename variables to be more meaningful BUG=chromium:1042953 TEST=verified locally. Change-Id: I1a452c2756491e69121a161a32ab007c4e79629d Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2006728 Reviewed-by: George Burgess Tested-by: Jian Cai --- llvm_tools/patch_manager_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm_tools/patch_manager_unittest.py b/llvm_tools/patch_manager_unittest.py index 688c2fe6..62947ed1 100755 --- a/llvm_tools/patch_manager_unittest.py +++ b/llvm_tools/patch_manager_unittest.py @@ -463,7 +463,7 @@ class PatchManagerTest(unittest.TestCase): # Simulate behavior for 'ApplyPatch()' when applying multiple applicable # patches. @CallCountsToMockFunctions - def MultipleCallsToApplyPatches(call_count, _, path_to_patch): + def MultipleCallsToApplyPatches(call_count, _src_path, path_to_patch): if call_count < 3: self.assertEqual( path_to_patch, @@ -569,7 +569,7 @@ class PatchManagerTest(unittest.TestCase): # Simulate behavior for 'ApplyPatch()' when applying multiple applicable # patches. @CallCountsToMockFunctions - def MultipleCallsToApplyPatches(call_count, _, path_to_patch): + def MultipleCallsToApplyPatches(call_count, _src_path, path_to_patch): if call_count < 3: self.assertEqual( path_to_patch, -- cgit v1.2.3 From 9a9fe3aa1c6b983619b9c7c8f3433dd76fd89aed Mon Sep 17 00:00:00 2001 From: chrome-bot Date: Mon, 20 Jan 2020 14:17:47 -0800 Subject: afdo_metadata: Publish new profiles for kernel 3.18. Update chromeos-kernel-3_18 from R81-12814.0-1578912053 to R81-12828.0-1579516702 Change-Id: Ib1a2799a3da46e6f695335c6a1d00d2142c4297c Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2011332 --- afdo_metadata/kernel_afdo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 350ac031..4b11efff 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -6,6 +6,6 @@ "name": "R81-12814.0-1578911615" }, "chromeos-kernel-3_18": { - "name": "R81-12814.0-1578912053" + "name": "R81-12828.0-1579516702" } } \ No newline at end of file -- cgit v1.2.3 From be0b3faefdbd5681ce49906246c30e117881a1ab Mon Sep 17 00:00:00 2001 From: chrome-bot Date: Tue, 21 Jan 2020 19:34:20 -0800 Subject: afdo_metadata: Publish new profiles for kernel 4.4. Update chromeos-kernel-4_4 from R81-12814.0-1578911615 to R81-12828.0-1579516426 Change-Id: I520e115bf5ebc19f71b21cfc66d887289df732cc Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2013699 --- afdo_metadata/kernel_afdo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 4b11efff..0cfffeaa 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -3,7 +3,7 @@ "name": "R81-12607.58-1578524353" }, "chromeos-kernel-4_4": { - "name": "R81-12814.0-1578911615" + "name": "R81-12828.0-1579516426" }, "chromeos-kernel-3_18": { "name": "R81-12828.0-1579516702" -- cgit v1.2.3 From c2dec89d0d9758bcbb2457a04f4bf49d5bf4731c Mon Sep 17 00:00:00 2001 From: Jian Cai Date: Thu, 16 Jan 2020 16:12:34 -0800 Subject: compiler_wrapper: add python search path Exporting BISECT_STAGE to POPULATE_GOOD currently makes the wrapper fail to build any files with the error message of "ModuleNotFoundError: No module named 'bisect_driver'". This patch fixes the issue by adding search path for bisect_driver.py explicitly. BUG=chromium:1042452 TEST=verified locally. Change-Id: I7c73459b377d9011c7087a9e241db378ffb216e8 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2003947 Tested-by: Jian Cai Reviewed-by: Manoj Gupta Reviewed-by: George Burgess --- compiler_wrapper/bisect_flag.go | 12 +++++++++++- compiler_wrapper/goldenutil_test.go | 7 +++++++ compiler_wrapper/testdata/android_golden/bisect.json | 9 +++++++++ compiler_wrapper/testdata/cros_clang_host_golden/bisect.json | 9 +++++++++ compiler_wrapper/testdata/cros_hardened_golden/bisect.json | 9 ++++++--- .../testdata/cros_hardened_llvmnext_golden/bisect.json | 9 ++++++--- .../testdata/cros_hardened_noccache_golden/bisect.json | 9 +++++++++ .../testdata/cros_nonhardened_golden/bisect.json | 9 ++++++--- 8 files changed, 63 insertions(+), 10 deletions(-) diff --git a/compiler_wrapper/bisect_flag.go b/compiler_wrapper/bisect_flag.go index 6271e23f..8aec3e7e 100644 --- a/compiler_wrapper/bisect_flag.go +++ b/compiler_wrapper/bisect_flag.go @@ -6,6 +6,7 @@ package main import ( "errors" + "os" "path/filepath" ) @@ -51,6 +52,15 @@ func calcBisectCommand(env env, cfg *config, bisectStage string, compilerCmd *co } } absCompilerPath := getAbsCmdPath(env, compilerCmd) + pythonPath, err := filepath.Abs(os.Args[0]) + if err != nil { + return nil, err + } + pythonPath, err = filepath.EvalSymlinks(pythonPath) + if err != nil { + return nil, err + } + pythonPath = filepath.Dir(pythonPath) return &command{ Path: "/usr/bin/env", Args: append([]string{ @@ -61,6 +71,6 @@ func calcBisectCommand(env env, cfg *config, bisectStage string, compilerCmd *co bisectDir, absCompilerPath, }, compilerCmd.Args...), - EnvUpdates: compilerCmd.EnvUpdates, + EnvUpdates: append(compilerCmd.EnvUpdates, "PYTHONPATH="+pythonPath), }, nil } diff --git a/compiler_wrapper/goldenutil_test.go b/compiler_wrapper/goldenutil_test.go index e6b42f77..23e003c5 100644 --- a/compiler_wrapper/goldenutil_test.go +++ b/compiler_wrapper/goldenutil_test.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" ) @@ -145,7 +146,13 @@ func fillGoldenResults(ctx *testContext, files []goldenFile) []goldenFile { // Create an empty wrapper at the given path. // Needed as we are resolving symlinks which stats the wrapper file. ctx.writeFile(record.WrapperCmd.Cmd.Path, "") + // Assign a fixed path to os.Args[0] to pass the test. + tmp := os.Args[0] + // callCompiler verifies os.Args[0] exists, so use a real file. + _, file, _, _ := runtime.Caller(1) + os.Args[0] = file record.WrapperCmd.ExitCode = callCompiler(ctx, ctx.cfg, record.WrapperCmd.Cmd) + os.Args[0] = tmp if hasInternalError(ctx.stderrString()) { ctx.t.Errorf("found an internal error for wrapperCmd %#v and env #%v. Got: %s", record.WrapperCmd.Cmd, record.Env, ctx.stderrString()) diff --git a/compiler_wrapper/testdata/android_golden/bisect.json b/compiler_wrapper/testdata/android_golden/bisect.json index a24222ab..be07da68 100644 --- a/compiler_wrapper/testdata/android_golden/bisect.json +++ b/compiler_wrapper/testdata/android_golden/bisect.json @@ -25,6 +25,9 @@ "/user/home/ANDROID_BISECT", "/tmp/stable/clang.real", "main.cc" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -57,6 +60,9 @@ "someBisectDir", "/tmp/stable/clang.real", "main.cc" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -92,6 +98,9 @@ "someBisectDir", "/tmp/stable/clang.real", "main.cc" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json index 5920a365..ad4feac3 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json @@ -39,6 +39,9 @@ "-Wno-unknown-warning-option", "main.cc", "-Wno-implicit-int-float-conversion" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -85,6 +88,9 @@ "-Wno-unknown-warning-option", "main.cc", "-Wno-implicit-int-float-conversion" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -134,6 +140,9 @@ "-Wno-unknown-warning-option", "main.cc", "-Wno-implicit-int-float-conversion" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index c1daaa88..6e1d4981 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -54,7 +54,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -116,7 +117,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -181,7 +183,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index c1daaa88..6e1d4981 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -54,7 +54,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -116,7 +117,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -181,7 +183,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index 46c29635..3fafcf01 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -48,6 +48,9 @@ "-B../../bin", "-target", "x86_64-cros-linux-gnu" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -103,6 +106,9 @@ "-B../../bin", "-target", "x86_64-cros-linux-gnu" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -161,6 +167,9 @@ "-B../../bin", "-target", "x86_64-cros-linux-gnu" + ], + "env_updates": [ + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index b97c379d..7d2acdac 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -46,7 +46,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -100,7 +101,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] } } @@ -157,7 +159,8 @@ "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu", "CCACHE_DIR=/var/cache/distfiles/ccache", "CCACHE_UMASK=002", - "CCACHE_CPP2=yes" + "CCACHE_CPP2=yes", + "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper" ] }, "stdout": "somemessage", -- cgit v1.2.3 From 0ee6a5701bedfa6b67b33f8b47cd705d713b1777 Mon Sep 17 00:00:00 2001 From: Zhizhou Yang Date: Wed, 22 Jan 2020 15:55:19 -0800 Subject: crosperf: remove AFE server interactions This patch removes afe locking mode for crosperf, since: 1) All lab machines has been migrated to skylab. 2) Interaction with afe depends on python 2 code in autotest. BUG=chromium:1044764 TEST=Passed unittest and simple experiment run. Change-Id: Ie0a250548d739f32bbc08619f637380104df6c28 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2016008 Commit-Queue: Zhizhou Yang Tested-by: Zhizhou Yang Auto-Submit: Zhizhou Yang Reviewed-by: George Burgess --- crosperf/experiment_runner.py | 5 +- lock_machine.py | 109 ++++-------------------------------------- 2 files changed, 11 insertions(+), 103 deletions(-) diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index a536f161..39e3f863 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -103,7 +103,7 @@ class ExperimentRunner(object): """Get where is the machine from. Returns: - The location of the machine: local, skylab or afe + The location of the machine: local or skylab """ # We assume that lab machine always starts with chromeos*, and local # machines are ip address. @@ -111,7 +111,7 @@ class ExperimentRunner(object): if lock_mgr.CheckMachineInSkylab(machine): return 'skylab' else: - return 'afe' + raise RuntimeError('Lab machine not in Skylab.') return 'local' def _LockAllMachines(self, experiment): @@ -122,7 +122,6 @@ class ExperimentRunner(object): from being able to update/use the machines while this experiment is running: - Skylab machines: Use skylab lease-dut mechanism to lease - - AFE machines: Use AFE lock mechanism to lock - Local machines: Use file lock mechanism to lock """ if test_flag.GetTestMode(): diff --git a/lock_machine.py b/lock_machine.py index eca4bd57..87230b70 100755 --- a/lock_machine.py +++ b/lock_machine.py @@ -39,17 +39,8 @@ class DontOwnLock(LockException): # This should not be raised if the user specified '--force' -class NoAFEServer(LockException): - """Raised when cannot find/access the autotest server.""" - - -class AFEAccessError(LockException): - """Raised when cannot get information about lab machine from lab server.""" - - class MachineType(enum.Enum): """Enum class to hold machine type.""" - AFE = 'afe' LOCAL = 'local' SKYLAB = 'skylab' @@ -58,14 +49,8 @@ class LockManager(object): """Class for locking/unlocking machines vie three different modes. This class contains methods for checking the locked status of machines, - and for changing the locked status. It handles HW lab machines (both AFE - and Skylab), and local machines, using appropriate locking mechanisms for - each. - - !!!IMPORTANT NOTE!!! The AFE server can only be called from the main - thread/process of a program. If you launch threads and try to call it - from a thread, you will get an error. This has to do with restrictions - in the Python virtual machine (and signal handling) and cannot be changed. + and for changing the locked status. It handles HW lab machines and local + machines, using appropriate locking mechanisms for each. """ SKYLAB_PATH = '/usr/local/bin/skylab' @@ -104,27 +89,11 @@ class LockManager(object): self.user = getpass.getuser() self.logger = log or logger.GetLogger() self.ce = command_executer.GetCommandExecuter(self.logger) - autotest_path = os.path.join(chromeos_root, - 'src/third_party/autotest/files') sys.path.append(chromeos_root) - sys.path.append(autotest_path) - sys.path.append(os.path.join(autotest_path, 'server', 'cros')) self.locks_dir = locks_dir - # We have to wait to do these imports until the paths above have - # been fixed. - # pylint: disable=import-error - from client import setup_modules - setup_modules.setup( - base_path=autotest_path, root_module_name='autotest_lib') - - from dynamic_suite import frontend_wrappers - - self.afe = frontend_wrappers.RetryingAFE( - timeout_min=30, delay_sec=10, debug=False, server='cautotest') - self.machines = list(set(remotes)) or [] self.toolchain_lab_machines = self.GetAllToolchainLabMachines() @@ -181,7 +150,6 @@ class LockManager(object): return MachineType.LOCAL if m in self.skylab_machines: return MachineType.SKYLAB - return MachineType.AFE def PrintStatusHeader(self): """Prints the status header lines for machines.""" @@ -196,8 +164,6 @@ class LockManager(object): state: A dictionary of the current state of the machine. machine_type: MachineType to determine where the machine is located. """ - if machine_type == MachineType.AFE and not m.endswith('.cros'): - m += '.cros' if state['locked']: print('%s (%s)\t\t%slocked by %s since %s' % (m, state['board'], '\t\t' if machine_type == MachineType.LOCAL else @@ -242,33 +208,6 @@ class LockManager(object): state = machine_states[m] self.PrintStatus(m, state, machine_type) - def UpdateLockInAFE(self, should_lock_machine, machine): - """Calls an AFE server to lock/unlock a machine. - - Args: - should_lock_machine: Boolean indicating whether to lock the machine (True) - or unlock the machine (False). - machine: The machine to update. - - Returns: - True if requested action succeeded, else False. - """ - kwargs = {'locked': should_lock_machine} - if should_lock_machine: - kwargs['lock_reason'] = 'toolchain user request (%s)' % self.user - - m = machine.split('.')[0] - afe_server = self.afe - - try: - afe_server.run( - 'modify_hosts', - host_filter_data={'hostname__in': [m]}, - update_data=kwargs) - except Exception: - return False - return True - def UpdateLockInSkylab(self, should_lock_machine, machine): """Ask skylab to lease/release a machine. @@ -333,8 +272,6 @@ class LockManager(object): ret = self.UpdateLockInSkylab(lock_machines, m) elif machine_type == MachineType.LOCAL: ret = self.UpdateFileLock(lock_machines, m) - else: - ret = self.UpdateLockInAFE(lock_machines, m) if ret: self.logger.LogOutput( @@ -412,48 +349,20 @@ class LockManager(object): Returns: A dictionary of machine states for all the machines in the LockManager object. - - Raises: - NoAFEServer: Cannot find the HW Lab AFE server. - AFEAccessError: An error occurred when querying the server about a - machine. """ - if not self.afe: - raise NoAFEServer('Error: Cannot connect to main AFE server.') - machine_list = {} for m in self.machines: # For local or skylab machines, we simply set {'locked': status} for them # TODO(zhizhouy): This is a quick fix since skylab cannot return host info # as afe does. We need to get more info such as locked_by when skylab # supports that. - if m in self.local_machines or m in self.skylab_machines: - values = { - 'locked': 0 if cmd == 'lock' else 1, - 'board': '??', - 'locked_by': '', - 'lock_time': '' - } - machine_list[m] = values - else: - # For autotest machines, we use afe APIs to get locking info. - mod_host = m.split('.')[0] - host_info = self.afe.get_hosts(hostname=mod_host) - if not host_info: - raise AFEAccessError('Unable to get information about %s from main' - ' autotest server.' % m) - host_info = host_info[0] - name = host_info.hostname - values = {} - values['board'] = host_info.platform if host_info.platform else '??' - values['locked'] = host_info.locked - if host_info.locked: - values['locked_by'] = host_info.locked_by - values['lock_time'] = host_info.lock_time - else: - values['locked_by'] = '' - values['lock_time'] = '' - machine_list[name] = values + values = { + 'locked': 0 if cmd == 'lock' else 1, + 'board': '??', + 'locked_by': '', + 'lock_time': '' + } + machine_list[m] = values self.ListMachineStates(machine_list) -- cgit v1.2.3 From 39a1c992ee085cac78f64b36b66daa1e1c8747ad Mon Sep 17 00:00:00 2001 From: Manoj Gupta Date: Thu, 23 Jan 2020 14:13:11 -0800 Subject: compiler_wrapper: strip newline char from resource dir. The resource dir value returned by calling clang has a new line character. This breaks running clang with "-Xclang-path=/path/to/" functionality as the resource directory with this new line does not exist. BUG=chromium:1044790 TEST=compilation with "-Xclang-path=/path/to/" works. Change-Id: I0ef31108719aed02a433aff3e751e961892dd15a Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2017844 Tested-by: Manoj Gupta Reviewed-by: George Burgess --- compiler_wrapper/clang_flags.go | 3 ++- compiler_wrapper/clang_flags_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go index 8b76e965..1f0e2231 100644 --- a/compiler_wrapper/clang_flags.go +++ b/compiler_wrapper/clang_flags.go @@ -172,7 +172,8 @@ func getClangResourceDir(env env, clangPath string) (string, error) { "failed to call clang to read the resouce-dir: %#v", readResourceCmd) } - return stdoutBuffer.String(), nil + resourceDir := strings.TrimRight(stdoutBuffer.String(), "\n") + return resourceDir, nil } // Return the a directory which contains an 'ld' that gcc is using. diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go index 0c0f1fdd..2a7fbd15 100644 --- a/compiler_wrapper/clang_flags_test.go +++ b/compiler_wrapper/clang_flags_test.go @@ -139,7 +139,7 @@ func TestUseXclangPathAndCalcResourceDirByNestedClangCall(t *testing.T) { if err := verifyArgOrder(cmd, "--print-resource-dir"); err != nil { t.Error(err) } - fmt.Fprint(stdout, "someResourcePath") + fmt.Fprint(stdout, "someResourcePath\n") return nil } cmd := ctx.must(callCompiler(ctx, ctx.cfg, -- cgit v1.2.3 From 1f29b1d8687d9a103bbbdb2609f70fa41cffba51 Mon Sep 17 00:00:00 2001 From: George Burgess IV Date: Mon, 27 Jan 2020 13:15:06 -0800 Subject: llvm_tools: remove unused import `requests` is an artifact of an old way of doing things, and breaks things on Android bots. Best to just remove it. This also fixes other misc lints in that file BUG=b:147699825 TEST=unittests Change-Id: I3f465f9a129679b104e328ab8307298af1df651b Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2023775 Reviewed-by: Manoj Gupta Reviewed-by: Pirama Arumuga Nainar Tested-by: George Burgess --- llvm_tools/get_llvm_hash.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py index 1864170d..a5b5429e 100755 --- a/llvm_tools/get_llvm_hash.py +++ b/llvm_tools/get_llvm_hash.py @@ -10,18 +10,14 @@ from __future__ import print_function import argparse import os -import re import shutil import subprocess import sys import tempfile -import git_llvm_rev - from contextlib import contextmanager -from subprocess_helpers import CheckCommand -from subprocess_helpers import check_output -import requests +import git_llvm_rev +from subprocess_helpers import CheckCommand, check_output _LLVM_GIT_URL = ('https://chromium.googlesource.com/external/github.com/llvm' '/llvm-project') @@ -33,6 +29,7 @@ def GetVersionFrom(src_dir, git_hash): """Obtain an SVN-style version number based on the LLVM git hash passed in. Args: + src_dir: LLVM's source directory. git_hash: The git hash. Returns: -- cgit v1.2.3