diff options
48 files changed, 258 insertions, 233 deletions
diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 0cfffeaa..967c3486 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -3,9 +3,9 @@ "name": "R81-12607.58-1578524353" }, "chromeos-kernel-4_4": { - "name": "R81-12828.0-1579516426" + "name": "R81-12861.0-1580726347" }, "chromeos-kernel-3_18": { - "name": "R81-12828.0-1579516702" + "name": "R81-12861.0-1580725953" } }
\ No newline at end of file diff --git a/binary_search_tool/bisect_driver.py b/binary_search_tool/bisect_driver.py index 6a69fbf4..82df15d6 100644 --- a/binary_search_tool/bisect_driver.py +++ b/binary_search_tool/bisect_driver.py @@ -5,6 +5,7 @@ # # pylint: disable=not-callable # pylint: disable=indentation + """Utilities for bisection of ChromeOS and Android object files. This module contains a set of utilities to allow bisection between @@ -25,6 +26,7 @@ import fcntl import os import shutil import subprocess +import stat import sys VALID_MODES = ('POPULATE_GOOD', 'POPULATE_BAD', 'TRIAGE') @@ -33,6 +35,8 @@ BAD_CACHE = 'bad' LIST_FILE = os.path.join(GOOD_CACHE, '_LIST') CONTINUE_ON_MISSING = os.environ.get('BISECT_CONTINUE_ON_MISSING', None) == '1' +CONTINUE_ON_REDUNDANCY = os.environ.get('BISECT_CONTINUE_ON_REDUNDANCY', + None) == '1' WRAPPER_SAFE_MODE = os.environ.get('BISECT_WRAPPER_SAFE_MODE', None) == '1' @@ -260,8 +264,13 @@ def cache_file(execargs, bisect_dir, cache, abs_file_path): with lock_file(os.path.join(population_dir, '_DUPS'), 'a') as dup_object_list: dup_object_list.write('%s\n' % abs_file_path) + if CONTINUE_ON_REDUNDANCY: + return True raise Exception( - 'Trying to cache file %s multiple times.' % abs_file_path) + 'Trying to cache file %s multiple times. To avoid the error, set \ + CONTINUE_ON_REDUNDANCY to 1. For reference, the list of such files \ + will be written to %s' % (abs_file_path, + os.path.join(population_dir, '_DUPS'))) shutil.copy2(abs_file_path, bisect_path) # Set cache object to be read-only so later compilations can't @@ -289,7 +298,10 @@ def restore_file(bisect_dir, cache, abs_file_path): if os.path.exists(cached_path): if os.path.exists(abs_file_path): os.remove(abs_file_path) - os.link(cached_path, abs_file_path) + shutil.copy2(cached_path, abs_file_path) + # Add write permission to the restored object files as some packages + # (such as kernels) may need write permission to delete files. + os.chmod(abs_file_path, os.stat(abs_file_path).st_mode | stat.S_IWUSR) else: raise Error(('%s is missing from %s cache! Unsure how to proceed. Make ' 'will now crash.' % (cache, cached_path))) diff --git a/binary_search_tool/common/test_setup.sh b/binary_search_tool/common/test_setup.sh index c4f5f698..56b8944a 100755 --- a/binary_search_tool/common/test_setup.sh +++ b/binary_search_tool/common/test_setup.sh @@ -94,6 +94,10 @@ echo if [[ "${BISECT_MODE}" == "OBJECT_MODE" ]]; then echo "EMERGING ${BISECT_PACKAGE}" + echo "sudo rm -rf /build/${BISECT_BOARD}/var/cache/portage/*" + sudo rm -rf /build/${BISECT_BOARD}/var/cache/portage/* + echo "sudo rm -rf /build/${BISECT_BOARD}/tmp/portage/${BISECT_PACKAGE}*" + sudo rm -rf /build/${BISECT_BOARD}/tmp/portage/${BISECT_PACKAGE}* CLEAN_DELAY=0 emerge-${BISECT_BOARD} -C ${BISECT_PACKAGE} emerge-${BISECT_BOARD} ${BISECT_PACKAGE} emerge_status=$? @@ -105,8 +109,15 @@ if [[ "${BISECT_MODE}" == "OBJECT_MODE" ]]; then echo echo "DEPLOYING" - echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}" - cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info + if [[ ${PACKAGE} == sys-kernel/chromeos-kernel-* ]]; then + echo "/mnt/host/source/src/scripts/update_kernel.sh \ + --remote=${BISECT_REMOTE}" + /mnt/host/source/src/scripts/update_kernel.sh --remote=${BISECT_REMOTE} + else + echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}" + cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info + fi + deploy_status=$? if [[ ${deploy_status} -eq 0 ]] ; then diff --git a/cros_utils/command_executer.py b/cros_utils/command_executer.py index c4d3fded..39bff5ed 100644..100755 --- a/cros_utils/command_executer.py +++ b/cros_utils/command_executer.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2011 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -17,11 +18,11 @@ import sys import tempfile import time -import logger -import misc +from cros_utils import logger mock_default = False +CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts' LOG_LEVEL = ('none', 'quiet', 'average', 'verbose') @@ -101,6 +102,7 @@ class CommandExecuter(object): # In this way the child cannot mess the parent's terminal. p = None try: + # pylint: disable=bad-option-value, subprocess-popen-preexec-fn p = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -238,11 +240,14 @@ class CommandExecuter(object): return command def WriteToTempShFile(self, contents): - handle, command_file = tempfile.mkstemp(prefix=os.uname()[1], suffix='.sh') - os.write(handle, '#!/bin/bash\n') - os.write(handle, contents) - os.close(handle) - return command_file + # TODO(crbug.com/1048938): use encoding='utf-8' when all dependencies have + # migrated to python 3. + with tempfile.NamedTemporaryFile( + 'w', delete=False, prefix=os.uname()[1], suffix='.sh') as f: + f.write('#!/bin/bash\n') + f.write(contents) + f.flush() + return f.name def CrosLearnBoard(self, chromeos_root, machine): command = self.RemoteAccessInitCommand(chromeos_root, machine) @@ -362,15 +367,20 @@ class CommandExecuter(object): if self.logger: self.logger.LogCmd(command, print_to_console=print_to_console) - handle, command_file = tempfile.mkstemp( + # TODO(crbug.com/1048938): use encoding='utf-8' when all dependencies have + # migrated to python 3. + with tempfile.NamedTemporaryFile( + 'w', + delete=False, dir=os.path.join(chromeos_root, 'src/scripts'), suffix='.sh', - prefix='in_chroot_cmd') - os.write(handle, '#!/bin/bash\n') - os.write(handle, command) - os.write(handle, '\n') - os.close(handle) + prefix='in_chroot_cmd') as f: + f.write('#!/bin/bash\n') + f.write(command) + f.write('\n') + f.flush() + command_file = f.name os.chmod(command_file, 0o777) # if return_output is set, run a dummy command first to make sure that @@ -386,7 +396,7 @@ class CommandExecuter(object): # Run command_file inside the chroot, making sure that any "~" is expanded # by the shell inside the chroot, not outside. command = ("cd %s; cros_sdk %s -- bash -c '%s/%s'" % - (chromeos_root, cros_sdk_options, misc.CHROMEOS_SCRIPTS_DIR, + (chromeos_root, cros_sdk_options, CHROMEOS_SCRIPTS_DIR, os.path.basename(command_file))) ret = self.RunCommandGeneric( command, @@ -601,6 +611,7 @@ class CommandExecuter(object): # In this way the child cannot mess the parent's terminal. pobject = None try: + # pylint: disable=bad-option-value, subprocess-popen-preexec-fn pobject = subprocess.Popen( cmd, cwd=cwd, diff --git a/cros_utils/email_sender.py b/cros_utils/email_sender.py index 3f724f64..0019982e 100755 --- a/cros_utils/email_sender.py +++ b/cros_utils/email_sender.py @@ -97,12 +97,14 @@ class EmailSender(object): if not text_to_send: text_to_send = 'Empty message body.' - body_fd, body_filename = tempfile.mkstemp() - to_be_deleted = [body_filename] + to_be_deleted = [] try: - os.write(body_fd, text_to_send) - os.close(body_fd) + with tempfile.NamedTemporaryFile( + 'w', encoding='utf-8', delete=False) as f: + f.write(text_to_send) + f.flush() + to_be_deleted.append(f.name) # Fix single-quotes inside the subject. In bash, to escape a single quote # (e.g 'don't') you need to replace it with '\'' (e.g. 'don'\''t'). To @@ -113,11 +115,10 @@ class EmailSender(object): if msg_type == 'html': command = ("sendgmr --to='%s' --from='%s' --subject='%s' " "--html_file='%s' --body_file=/dev/null" % - (to_list, email_from, subject, body_filename)) + (to_list, email_from, subject, f.name)) else: - command = ( - "sendgmr --to='%s' --from='%s' --subject='%s' " - "--body_file='%s'" % (to_list, email_from, subject, body_filename)) + command = ("sendgmr --to='%s' --from='%s' --subject='%s' " + "--body_file='%s'" % (to_list, email_from, subject, f.name)) if email_cc: cc_list = ','.join(email_cc) @@ -133,10 +134,11 @@ class EmailSender(object): report_suffix = '_report.html' else: report_suffix = '_report.txt' - fd, fname = tempfile.mkstemp(suffix=report_suffix) - os.write(fd, attachment.content) - os.close(fd) - attachment_files.append(fname) + with tempfile.NamedTemporaryFile( + 'w', encoding='utf-8', delete=False, suffix=report_suffix) as f: + f.write(attachment.content) + f.flush() + attachment_files.append(f.name) files = ','.join(attachment_files) command += " --attachment_files='%s'" % files to_be_deleted += attachment_files diff --git a/cros_utils/file_utils.py b/cros_utils/file_utils.py index fe6fc16f..f0e4064c 100644 --- a/cros_utils/file_utils.py +++ b/cros_utils/file_utils.py @@ -10,7 +10,8 @@ from __future__ import print_function import errno import os import shutil -import command_executer + +from cros_utils import command_executer class FileUtils(object): @@ -65,7 +66,7 @@ class FileUtils(object): shutil.rmtree(path, ignore_errors=True) def WriteFile(self, path, contents): - with open(path, 'wb') as f: + with open(path, 'w', encoding='utf-8') as f: f.write(contents) diff --git a/cros_utils/locks.py b/cros_utils/locks.py index 4ecbe0a9..848e23fc 100644 --- a/cros_utils/locks.py +++ b/cros_utils/locks.py @@ -12,7 +12,7 @@ import time import lock_machine -import logger +from cros_utils import logger def AcquireLock(machines, chromeos_root, timeout=1200): diff --git a/cros_utils/misc.py b/cros_utils/misc.py index 48e3c724..246767f0 100644 --- a/cros_utils/misc.py +++ b/cros_utils/misc.py @@ -17,10 +17,10 @@ import shutil import sys import traceback -import command_executer -import logger +from cros_utils import command_executer +from cros_utils import logger -CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts' +CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts' TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/' 'cros_utils/toolchain_utils.sh') diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py index 9527fde0..300c2d79 100644 --- a/cros_utils/tabulator.py +++ b/cros_utils/tabulator.py @@ -72,8 +72,8 @@ import sys import numpy import scipy -from email_sender import EmailSender -import misc +from cros_utils.email_sender import EmailSender +from cros_utils import misc def _AllFloat(values): diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py index f17de1be..74d461a6 100644 --- a/crosperf/benchmark_run.py +++ b/crosperf/benchmark_run.py @@ -131,7 +131,7 @@ class BenchmarkRun(threading.Thread): self.failure_reason = 'Return value of test suite was non-zero.' self.timeline.Record(STATUS_FAILED) - except Exception, e: + except Exception as e: self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) traceback.print_exc() if self.timeline.GetLastEvent() != STATUS_FAILED: diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py index 5696c107..4cadc35c 100755 --- a/crosperf/benchmark_run_unittest.py +++ b/crosperf/benchmark_run_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. @@ -9,10 +9,9 @@ from __future__ import print_function -import unittest import inspect - -import mock +import unittest +import unittest.mock as mock import benchmark_run @@ -121,7 +120,7 @@ class BenchmarkRunTest(unittest.TestCase): 'machine_manager', 'logger_to_use', 'log_level', 'share_cache', 'dut_config' ] - arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__) + arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__) self.assertEqual(len(arg_spec.args), len(args_list)) self.assertEqual(arg_spec.args, args_list) diff --git a/crosperf/benchmark_unittest.py b/crosperf/benchmark_unittest.py index 63c0a1c0..70508b19 100755 --- a/crosperf/benchmark_unittest.py +++ b/crosperf/benchmark_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2014 The Chromium OS Authors. All rights reserved. @@ -62,7 +62,7 @@ class BenchmarkTestCase(unittest.TestCase): 'perf_args', 'suite', 'show_all_results', 'retries', 'run_local', 'cwp_dso', 'weight' ] - arg_spec = inspect.getargspec(Benchmark.__init__) + arg_spec = inspect.getfullargspec(Benchmark.__init__) self.assertEqual(len(arg_spec.args), len(args_list)) for arg in args_list: self.assertIn(arg, arg_spec.args) diff --git a/crosperf/config_unittest.py b/crosperf/config_unittest.py index 9e71539d..208f44dc 100755 --- a/crosperf/config_unittest.py +++ b/crosperf/config_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2014 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/crosperf/crosperf b/crosperf/crosperf index a29dcbfa..c98f2dd4 100755 --- a/crosperf/crosperf +++ b/crosperf/crosperf @@ -1,2 +1,7 @@ #!/bin/bash -PYTHONPATH=$(dirname $0)/..:$PYTHONPATH exec python $(dirname $0)/crosperf.py "$@" +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +PYTHONPATH=$(dirname "$0")/..:${PYTHONPATH} exec \ +python3 "$(dirname "$0")/crosperf.py" "$@" diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py index 75494371..ec07e7c7 100755 --- a/crosperf/crosperf.py +++ b/crosperf/crosperf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2011 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -110,7 +110,7 @@ def RunCrosperf(argv): test_flag.SetTestMode(True) experiment_file = ExperimentFile( - open(experiment_filename, 'rb'), option_settings) + open(experiment_filename, encoding='utf-8'), option_settings) if not experiment_file.GetGlobalSettings().GetField('name'): experiment_name = os.path.basename(experiment_filename) experiment_file.GetGlobalSettings().SetField('name', experiment_name) diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index f7ffa1b7..ffd964a2 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. @@ -12,11 +12,9 @@ from __future__ import print_function import argparse import io -import os import tempfile import unittest - -import mock +import unittest.mock as mock import crosperf import settings_factory @@ -46,14 +44,14 @@ class CrosperfTest(unittest.TestCase): """Crosperf test class.""" def setUp(self): - input_file = io.BytesIO(EXPERIMENT_FILE_1) + input_file = io.StringIO(EXPERIMENT_FILE_1) self.exp_file = experiment_file.ExperimentFile(input_file) def testDryRun(self): - filehandle, filename = tempfile.mkstemp() - os.write(filehandle, EXPERIMENT_FILE_1) - crosperf.Main(['', filename, '--dry_run']) - os.remove(filename) + with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f: + f.write(EXPERIMENT_FILE_1) + f.flush() + crosperf.Main(['', f.name, '--dry_run']) def testConvertOptionsToSettings(self): parser = argparse.ArgumentParser() diff --git a/crosperf/download_images.py b/crosperf/download_images.py index f855cb60..9bd4a8bf 100644 --- a/crosperf/download_images.py +++ b/crosperf/download_images.py @@ -56,8 +56,8 @@ class ImageDownloader(object): def GetBuildID(self, chromeos_root, xbuddy_label): # Get the translation of the xbuddy_label into the real Google Storage # image name. - command = ('cd ~/trunk/src/third_party/toolchain-utils/crosperf; ' - "python translate_xbuddy.py '%s'" % xbuddy_label) + command = ('cd /mnt/host/source/src/third_party/toolchain-utils/crosperf; ' + "./translate_xbuddy.py '%s'" % xbuddy_label) _, build_id_tuple_str, _ = self._ce.ChrootRunCommandWOutput( chromeos_root, command) if not build_id_tuple_str: @@ -226,7 +226,7 @@ class ImageDownloader(object): status = self.VerifyFileExists(chromeos_root, build_id, autotest_packages_name) if status != 0: - default_autotest_dir = '~/trunk/src/third_party/autotest/files' + default_autotest_dir = '/mnt/host/source/src/third_party/autotest/files' print( '(Warning: Could not find autotest packages .)\n' '(Warning: Defaulting autotest path to %s .' % default_autotest_dir) diff --git a/crosperf/download_images_buildid_test.py b/crosperf/download_images_buildid_test.py index a376691c..fc37f2c1 100755 --- a/crosperf/download_images_buildid_test.py +++ b/crosperf/download_images_buildid_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2014 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py index 94504d3f..62b8d891 100755 --- a/crosperf/download_images_unittest.py +++ b/crosperf/download_images_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -10,8 +10,7 @@ from __future__ import print_function import os import unittest - -import mock +import unittest.mock as mock import download_images from cros_utils import command_executer diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 69e34fcc..f5b17ee2 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. @@ -13,8 +13,7 @@ import io import os import socket import unittest - -import mock +import unittest.mock as mock from cros_utils import command_executer from cros_utils.file_utils import FileUtils @@ -85,7 +84,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.append_benchmark_call_args = [] def testLoadExperimentFile1(self): - experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1)) + experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1)) exp = ExperimentFactory().GetExperiment( experiment_file, working_directory='', log_dir='') self.assertEqual(exp.remote, ['chromeos-alex3']) @@ -104,7 +103,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertEqual(exp.labels[0].board, 'x86-alex') def testLoadExperimentFile2CWP(self): - experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_2)) + experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_2)) exp = ExperimentFactory().GetExperiment( experiment_file, working_directory='', log_dir='') self.assertEqual(exp.cwp_dso, 'kallsyms') @@ -113,7 +112,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertEqual(exp.benchmarks[1].weight, 0.2) def testDuplecateBenchmark(self): - mock_experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1)) + mock_experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1)) mock_experiment_file.all_settings = [] benchmark_settings1 = settings_factory.BenchmarkSettings('name') mock_experiment_file.all_settings.append(benchmark_settings1) @@ -125,7 +124,7 @@ class ExperimentFactoryTest(unittest.TestCase): ef.GetExperiment(mock_experiment_file, '', '') def testCWPExceptions(self): - mock_experiment_file = ExperimentFile(io.BytesIO('')) + mock_experiment_file = ExperimentFile(io.StringIO('')) mock_experiment_file.all_settings = [] global_settings = settings_factory.GlobalSettings('test_name') global_settings.SetField('locks_dir', '/tmp') @@ -305,7 +304,7 @@ class ExperimentFactoryTest(unittest.TestCase): label_settings.GetXbuddyPath = FakeGetXbuddyPath - mock_experiment_file = ExperimentFile(io.BytesIO('')) + mock_experiment_file = ExperimentFile(io.StringIO('')) mock_experiment_file.all_settings = [] test_flag.SetTestMode(True) @@ -332,7 +331,7 @@ class ExperimentFactoryTest(unittest.TestCase): # First test. General test. exp = ef.GetExperiment(mock_experiment_file, '', '') - self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80']) + self.assertCountEqual(exp.remote, ['123.45.67.89', '123.45.76.80']) self.assertEqual(exp.cache_conditions, [0, 2, 1]) self.assertEqual(exp.log_level, 'average') @@ -354,9 +353,9 @@ class ExperimentFactoryTest(unittest.TestCase): test_flag.SetTestMode(True) label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros') exp = ef.GetExperiment(mock_experiment_file, '', '') - self.assertEqual( + self.assertCountEqual( exp.remote, - ['chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80']) + ['123.45.67.89', '123.45.76.80', 'chromeos1.cros', 'chromeos2.cros']) # Third test: Automatic fixing of bad logging_level param: global_settings.SetField('logging_level', 'really loud!') @@ -392,7 +391,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertEqual(len(exp.labels), 2) self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path') self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path') - self.assertEqual( + self.assertCountEqual( exp.remote, ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']) @@ -431,7 +430,7 @@ class ExperimentFactoryTest(unittest.TestCase): ef.CheckSkylabTool(chromeos_root, log_level) self.assertEqual(mock_runcmd.call_count, 1) self.assertEqual( - err.exception.message, 'Skylab tool not installed ' + str(err.exception), 'Skylab tool not installed ' 'correctly, please try to manually install it from ' '/tmp/chromeos/chromeos-admin/lab-tools/setup_lab_tools') diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py index 33459d11..1d89edad 100644 --- a/crosperf/experiment_file.py +++ b/crosperf/experiment_file.py @@ -129,7 +129,7 @@ class ExperimentFile(object): self.global_settings.SetField(field[0], field[1], field[2]) else: raise IOError('Unexpected line.') - except Exception, err: + except Exception as err: raise RuntimeError('Line %d: %s\n==> %s' % (reader.LineNo(), str(err), reader.CurrentLine(False))) diff --git a/crosperf/experiment_file_unittest.py b/crosperf/experiment_file_unittest.py index 12b68223..0d4e1e67 100755 --- a/crosperf/experiment_file_unittest.py +++ b/crosperf/experiment_file_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. @@ -147,13 +147,13 @@ benchmark: PageCycler { } label: image1 { -\tremote: chromeos-alex3 \tchromeos_image: /usr/local/google/cros_image1.bin +\tremote: chromeos-alex3 } label: image2 { -\tremote: chromeos-lumpy1 \tchromeos_image: /usr/local/google/cros_image2.bin +\tremote: chromeos-lumpy1 }\n\n""" @@ -161,7 +161,7 @@ class ExperimentFileTest(unittest.TestCase): """The main class for Experiment File test.""" def testLoadExperimentFile1(self): - input_file = io.BytesIO(EXPERIMENT_FILE_1) + input_file = io.StringIO(EXPERIMENT_FILE_1) experiment_file = ExperimentFile(input_file) global_settings = experiment_file.GetGlobalSettings() self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3']) @@ -181,7 +181,7 @@ class ExperimentFileTest(unittest.TestCase): self.assertEqual(label_settings[0].GetField('remote'), ['chromeos-alex3']) def testOverrideSetting(self): - input_file = io.BytesIO(EXPERIMENT_FILE_2) + input_file = io.StringIO(EXPERIMENT_FILE_2) experiment_file = ExperimentFile(input_file) global_settings = experiment_file.GetGlobalSettings() self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3']) @@ -194,11 +194,11 @@ class ExperimentFileTest(unittest.TestCase): self.assertEqual(benchmark_settings[1].GetField('iterations'), 2) def testDuplicateLabel(self): - input_file = io.BytesIO(EXPERIMENT_FILE_3) + input_file = io.StringIO(EXPERIMENT_FILE_3) self.assertRaises(Exception, ExperimentFile, input_file) def testDuplicateBenchmark(self): - input_file = io.BytesIO(EXPERIMENT_FILE_4) + input_file = io.StringIO(EXPERIMENT_FILE_4) experiment_file = ExperimentFile(input_file) benchmark_settings = experiment_file.GetSettings('benchmark') self.assertEqual(benchmark_settings[0].name, 'webrtc') @@ -209,13 +209,13 @@ class ExperimentFileTest(unittest.TestCase): '--story-tag-filter=smoothness') def testCanonicalize(self): - input_file = io.BytesIO(EXPERIMENT_FILE_1) + input_file = io.StringIO(EXPERIMENT_FILE_1) experiment_file = ExperimentFile(input_file) res = experiment_file.Canonicalize() self.assertEqual(res, OUTPUT_FILE) def testLoadDutConfigExperimentFile_Good(self): - input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD) + input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD) experiment_file = ExperimentFile(input_file) global_settings = experiment_file.GetGlobalSettings() self.assertEqual(global_settings.GetField('turbostat'), False) @@ -228,22 +228,21 @@ class ExperimentFileTest(unittest.TestCase): self.assertEqual(global_settings.GetField('top_interval'), 5) def testLoadDutConfigExperimentFile_WrongGovernor(self): - input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV) + input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV) with self.assertRaises(RuntimeError) as msg: ExperimentFile(input_file) - self.assertRegexpMatches( - str(msg.exception), 'governor: misspelled_governor') - self.assertRegexpMatches( + self.assertRegex(str(msg.exception), 'governor: misspelled_governor') + self.assertRegex( str(msg.exception), "Invalid enum value for field 'governor'." r' Must be one of \(performance, powersave, userspace, ondemand,' r' conservative, schedutils, sched, interactive\)') def testLoadDutConfigExperimentFile_WrongCpuUsage(self): - input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE) + input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE) with self.assertRaises(RuntimeError) as msg: ExperimentFile(input_file) - self.assertRegexpMatches(str(msg.exception), 'cpu_usage: unknown') - self.assertRegexpMatches( + self.assertRegex(str(msg.exception), 'cpu_usage: unknown') + self.assertRegex( str(msg.exception), "Invalid enum value for field 'cpu_usage'." r' Must be one of \(all, big_only, little_only, exclusive_cores\)') diff --git a/crosperf/experiment_runner_unittest.py b/crosperf/experiment_runner_unittest.py index 8747de45..4905975c 100755 --- a/crosperf/experiment_runner_unittest.py +++ b/crosperf/experiment_runner_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. @@ -15,7 +15,7 @@ import os import time import unittest -import mock +import unittest.mock as mock import experiment_runner import experiment_status @@ -109,7 +109,7 @@ class ExperimentRunnerTest(unittest.TestCase): def make_fake_experiment(self): test_flag.SetTestMode(True) - experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1)) + experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1)) experiment = ExperimentFactory().GetExperiment( experiment_file, working_directory='', log_dir='') return experiment @@ -409,7 +409,7 @@ class ExperimentRunnerTest(unittest.TestCase): @mock.patch.object(Result, 'CopyResultsTo') @mock.patch.object(Result, 'CleanUp') @mock.patch.object(Result, 'FormatStringTop5') - @mock.patch('__builtin__.open', new_callable=mock.mock_open) + @mock.patch('builtins.open', new_callable=mock.mock_open) def test_store_results(self, mock_open, mock_top5, mock_cleanup, mock_copy, _mock_text_report, mock_report, mock_writefile, mock_mkdir, mock_rmdir): diff --git a/crosperf/flag_test_unittest.py b/crosperf/flag_test_unittest.py index cb0e59e6..1e77c8a5 100755 --- a/crosperf/flag_test_unittest.py +++ b/crosperf/flag_test_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2014 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/crosperf/generate_report.py b/crosperf/generate_report.py index cea95a2e..bae365dc 100755 --- a/crosperf/generate_report.py +++ b/crosperf/generate_report.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2016 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -104,7 +104,7 @@ def CutResultsInPlace(results, max_keys=50, complain_on_update=True): } """ actually_updated = False - for bench_results in results.itervalues(): + for bench_results in results.values(): for platform_results in bench_results: for i, result in enumerate(platform_results): # Keep the keys that come earliest when sorted alphabetically. @@ -127,26 +127,6 @@ def CutResultsInPlace(results, max_keys=50, complain_on_update=True): return results -def _ConvertToASCII(obj): - """Convert an object loaded from JSON to ASCII; JSON gives us unicode.""" - - # In python 3, we can directly return the unicode loaded from json. - if sys.version_info.major == 3: - return obj - # TODO(zhizhouy): Drop the following code once migrated to python 3. - # Using something like `object_hook` is insufficient, since it only fires on - # actual JSON objects. `encoding` fails, too, since the default decoder always - # uses unicode() to decode strings. - # pylint: disable=unicode-builtin - if isinstance(obj, unicode): - return str(obj) - if isinstance(obj, dict): - return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()} - if isinstance(obj, list): - return [_ConvertToASCII(v) for v in obj] - return obj - - def _PositiveInt(s): i = int(s) if i < 0: @@ -279,10 +259,8 @@ def _ParseArgs(argv): def Main(argv): args = _ParseArgs(argv) - # JSON likes to load UTF-8; our results reporter *really* doesn't like - # UTF-8. with PickInputFile(args.input) as in_file: - raw_results = _ConvertToASCII(json.load(in_file)) + raw_results = json.load(in_file) platform_names = raw_results['platforms'] results = raw_results['data'] diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py index 465db29b..e19d4695 100755 --- a/crosperf/generate_report_unittest.py +++ b/crosperf/generate_report_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2016 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -12,7 +12,7 @@ from __future__ import print_function import copy import json import unittest -import mock +import unittest.mock as mock import generate_report import results_report @@ -49,7 +49,7 @@ class GenerateReportTests(unittest.TestCase): } results = generate_report.CountBenchmarks(runs) expected_results = [('foo', 4), ('bar', 0), ('baz', 3)] - self.assertEqual(sorted(expected_results), sorted(results)) + self.assertCountEqual(expected_results, results) def testCutResultsInPlace(self): bench_data = { @@ -82,8 +82,8 @@ class GenerateReportTests(unittest.TestCase): bench_data, max_keys=max_keys, complain_on_update=False) # Cuts should be in-place. self.assertIs(results, bench_data) - self.assertEqual( - sorted(original_bench_data.keys()), sorted(bench_data.keys())) + self.assertCountEqual( + list(original_bench_data.keys()), list(bench_data.keys())) for bench_name, original_runs in original_bench_data.items(): bench_runs = bench_data[bench_name] self.assertEqual(len(original_runs), len(bench_runs)) @@ -135,13 +135,11 @@ class GenerateReportTests(unittest.TestCase): self.assertEqual(0, return_code) self.assertEqual(mock_run_actions.call_count, 1) ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] - self.assertEqual( - sorted(ctors), - sorted([ - results_report.JSONResultsReport, - results_report.TextResultsReport, - results_report.HTMLResultsReport, - ])) + self.assertEqual(ctors, [ + results_report.JSONResultsReport, + results_report.TextResultsReport, + results_report.HTMLResultsReport, + ]) @mock.patch('generate_report.RunActions') def testMainSelectsHTMLIfNoReportsGiven(self, mock_run_actions): diff --git a/crosperf/label.py b/crosperf/label.py index bebc2706..b8122613 100644 --- a/crosperf/label.py +++ b/crosperf/label.py @@ -93,7 +93,8 @@ class Label(object): if self.image_type == 'local': self.checksum = ImageChecksummer().Checksum(self, self.log_level) elif self.image_type == 'trybot': - self.checksum = hashlib.md5(self.chromeos_image).hexdigest() + self.checksum = hashlib.md5( + self.chromeos_image.encode('utf-8')).hexdigest() def _GetImageType(self, chromeos_image): image_type = None diff --git a/crosperf/machine_image_manager_unittest.py b/crosperf/machine_image_manager_unittest.py index 062c185f..fbbca7b6 100755 --- a/crosperf/machine_image_manager_unittest.py +++ b/crosperf/machine_image_manager_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2015 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -53,13 +53,6 @@ class MachineImageManagerTester(unittest.TestCase): duts.append(MockDut(n)) return duts - def print_matrix(self, matrix): - # pylint: disable=expression-not-assigned - for r in matrix: - for v in r: - print('{} '.format('.' if v == ' ' else v)), - print('') - def create_labels_and_duts_from_pattern(self, pattern): labels = [] duts = [] diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index 6ed7e513..7211662c 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -151,9 +151,8 @@ class CrosMachine(object): def _GetMD5Checksum(self, ss): if ss: - return hashlib.md5(ss).hexdigest() - else: - return '' + return hashlib.md5(ss.encode('utf-8')).hexdigest() + return '' def _GetMachineID(self): command = 'dump_vpd_log --full --stdout' diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py index f6a77bb8..26eacbd7 100755 --- a/crosperf/machine_manager_unittest.py +++ b/crosperf/machine_manager_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. @@ -13,8 +13,7 @@ import os.path import time import hashlib import unittest - -import mock +import unittest.mock as mock import label import machine_manager @@ -778,7 +777,7 @@ class CrosMachineTest(unittest.TestCase): self.assertEqual(cm.phys_kbytes, 4194304) mock_run_cmd.return_value = [1, MEMINFO_STRING, ''] - self.assertRaises(cm._GetMemoryInfo) + self.assertRaises(Exception, cm._GetMemoryInfo) @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput') @mock.patch.object(machine_manager.CrosMachine, 'SetUpChecksumInfo') @@ -838,7 +837,7 @@ class CrosMachineTest(unittest.TestCase): '44:6d:57:20:4a:c5 txqueuelen 1000 (Ethernet)') mock_run_cmd.return_value = [0, 'invalid hardware config', ''] - self.assertRaises(cm._GetMachineID) + self.assertRaises(Exception, cm._GetMachineID) def test_add_cooldown_waittime(self): cm = machine_manager.CrosMachine('1.2.3.4.cros', '/usr/local/chromeos', diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index bd80e8a7..98062194 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -198,8 +198,8 @@ class Result(object): command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) self.ce.RunCommand(command, print_to_console=False) - command = ('python generate_test_report --no-color --csv %s' % - (os.path.join('/tmp', os.path.basename(self.temp_dir)))) + command = ('./generate_test_report --no-color --csv %s' % (os.path.join( + '/tmp', os.path.basename(self.temp_dir)))) _, out, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command, print_to_console=False) keyvals_dict = {} @@ -869,7 +869,7 @@ class Result(object): self.suite = suite self.cwp_dso = cwp_dso # Read in everything from the cache directory. - with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: + with open(os.path.join(cache_dir, RESULTS_FILE), 'rb') as f: self.out = pickle.load(f) self.err = pickle.load(f) self.retval = pickle.load(f) @@ -907,7 +907,7 @@ class Result(object): temp_dir = tempfile.mkdtemp() # Store to the temp directory. - with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: + with open(os.path.join(temp_dir, RESULTS_FILE), 'wb') as f: pickle.dump(self.out, f) pickle.dump(self.err, f) pickle.dump(self.retval, f) @@ -1040,7 +1040,7 @@ class TelemetryResult(Result): self.test_name = test self.suite = suite self.cwp_dso = cwp_dso - with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: + with open(os.path.join(cache_dir, RESULTS_FILE), 'rb') as f: self.out = pickle.load(f) self.err = pickle.load(f) self.retval = pickle.load(f) @@ -1183,7 +1183,8 @@ class ResultsCache(object): if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: checksum = '*' elif self.label.image_type == 'trybot': - checksum = hashlib.md5(self.label.chromeos_image).hexdigest() + checksum = hashlib.md5( + self.label.chromeos_image.encode('utf-8')).hexdigest() elif self.label.image_type == 'official': checksum = '*' else: @@ -1192,7 +1193,8 @@ class ResultsCache(object): if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: image_path_checksum = '*' else: - image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() + image_path_checksum = hashlib.md5( + self.chromeos_image.encode('utf-8')).hexdigest() machine_id_checksum = '' if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: @@ -1208,7 +1210,7 @@ class ResultsCache(object): temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, self.run_local) - test_args_checksum = hashlib.md5(temp_test_args).hexdigest() + test_args_checksum = hashlib.md5(temp_test_args.encode('utf-8')).hexdigest() return (image_path_checksum, self.test_name, str(self.iteration), test_args_checksum, checksum, machine_checksum, machine_id_checksum, str(self.CACHE_VERSION)) diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index c1c27265..1e7f04a1 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. @@ -13,8 +13,7 @@ import os import shutil import tempfile import unittest - -import mock +import unittest.mock as mock import image_checksummer import machine_manager @@ -236,13 +235,13 @@ TOP_DATA = [ 'top5': [2.0], }, { - 'cmd': 'sshd', + 'cmd': 'spi5', 'cpu_avg': 0.5, 'count': 1, 'top5': [1.0], }, { - 'cmd': 'spi5', + 'cmd': 'sshd', 'cpu_avg': 0.5, 'count': 1, 'top5': [1.0], @@ -689,8 +688,7 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_chrootruncmd.call_count, 1) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], - ('/tmp', - ('python generate_test_report --no-color --csv %s') % TMP_DIR1)) + ('/tmp', ('./generate_test_report --no-color --csv %s') % TMP_DIR1)) self.assertEqual(mock_getpath.call_count, 1) self.assertEqual(mock_mkdtemp.call_count, 1) self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) @@ -893,7 +891,7 @@ class ResultTest(unittest.TestCase): def test_process_turbostat_results_with_valid_data(self): """Normal case when log exists and contains valid data.""" self.result.turbostat_log_file = '/tmp/somelogfile.log' - with mock.patch('__builtin__.open', + with mock.patch('builtins.open', mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT)) as mo: cpustats = self.result.ProcessTurbostatResults() # Check that the log got opened and data were read/parsed. @@ -904,7 +902,7 @@ class ResultTest(unittest.TestCase): def test_process_turbostat_results_from_empty_file(self): """Error case when log exists but file is empty.""" self.result.turbostat_log_file = '/tmp/emptylogfile.log' - with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo: + with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo: cpustats = self.result.ProcessTurbostatResults() # Check that the log got opened and parsed successfully and empty data # returned. @@ -932,7 +930,7 @@ class ResultTest(unittest.TestCase): returned cpustats. """ self.result.cpustats_log_file = '/tmp/somelogfile.log' - with mock.patch('__builtin__.open', + with mock.patch('builtins.open', mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT)) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and data were read/parsed. @@ -949,7 +947,7 @@ class ResultTest(unittest.TestCase): returned cpustats. """ self.result.cpustats_log_file = '/tmp/somelogfile.log' - with mock.patch('__builtin__.open', + with mock.patch('builtins.open', mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT)) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and data were read/parsed. @@ -960,7 +958,7 @@ class ResultTest(unittest.TestCase): def test_process_cpustats_results_from_empty_file(self): """Error case when log exists but file is empty.""" self.result.cpustats_log_file = '/tmp/emptylogfile.log' - with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo: + with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and parsed successfully and empty data # returned. @@ -972,8 +970,7 @@ class ResultTest(unittest.TestCase): """Process top log with valid data.""" self.result.top_log_file = '/tmp/fakelogfile.log' - with mock.patch('__builtin__.open', - mock.mock_open(read_data=TOP_LOG)) as mo: + with mock.patch('builtins.open', mock.mock_open(read_data=TOP_LOG)) as mo: topproc = self.result.ProcessTopResults() # Check that the log got opened and data were read/parsed. calls = [mock.call('/tmp/fakelogfile.log')] @@ -983,7 +980,7 @@ class ResultTest(unittest.TestCase): def test_process_top_results_from_empty_file(self): """Error case when log exists but file is empty.""" self.result.top_log_file = '/tmp/emptylogfile.log' - with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo: + with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo: topcalls = self.result.ProcessTopResults() # Check that the log got opened and parsed successfully and empty data # returned. diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py index 4879caeb..674745fb 100644 --- a/crosperf/results_organizer.py +++ b/crosperf/results_organizer.py @@ -80,7 +80,7 @@ def _GetNonDupLabel(max_dup, runs): # pylint: disable=cell-var-from-loop added_runs = _Repeat( lambda: _DictWithReturnValues(run_retval, run_pass_fail), max_dup) - for key, value in run.iteritems(): + for key, value in run.items(): match = _DUP_KEY_REGEX.match(key) if not match: new_run[key] = value @@ -94,7 +94,7 @@ def _GetNonDupLabel(max_dup, runs): def _DuplicatePass(result, benchmarks): """Properly expands keys like `foo{1}` in `result`.""" - for bench, data in result.iteritems(): + for bench, data in result.items(): max_dup = _GetMaxDup(data) # If there's nothing to expand, there's nothing to do. if not max_dup: diff --git a/crosperf/results_organizer_unittest.py b/crosperf/results_organizer_unittest.py index 39a8cce6..f259879d 100755 --- a/crosperf/results_organizer_unittest.py +++ b/crosperf/results_organizer_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. diff --git a/crosperf/results_report.py b/crosperf/results_report.py index 81591db7..ff6c4f96 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -100,10 +100,10 @@ def _FilterPerfReport(event_threshold, report): def filter_dict(m): return { - fn_name: pct for fn_name, pct in m.iteritems() if pct >= event_threshold + fn_name: pct for fn_name, pct in m.items() if pct >= event_threshold } - return {event: filter_dict(m) for event, m in report.iteritems()} + return {event: filter_dict(m) for event, m in report.items()} class _PerfTable(object): @@ -186,7 +186,7 @@ def _GetTables(benchmark_results, columns, table_type): iter_counts = benchmark_results.iter_counts result = benchmark_results.run_keyvals tables = [] - for bench_name, runs in result.iteritems(): + for bench_name, runs in result.items(): iterations = iter_counts[bench_name] ben_table = _GetResultsTableHeader(bench_name, iterations) @@ -438,7 +438,7 @@ class TextResultsReport(ResultsReport): def _GetHTMLCharts(label_names, test_results): charts = [] - for item, runs in test_results.iteritems(): + for item, runs in test_results.items(): # Fun fact: label_names is actually *entirely* useless as a param, since we # never add headers. We still need to pass it anyway. table = TableGenerator(runs, label_names).GetTable() @@ -734,7 +734,7 @@ class JSONResultsReport(ResultsReport): label_names = benchmark_results.label_names summary_field_defaults = self.summary_field_defaults final_results = [] - for test, test_results in benchmark_results.run_keyvals.iteritems(): + for test, test_results in benchmark_results.run_keyvals.items(): for label_name, label_results in zip(label_names, test_results): for iter_results in label_results: passed = iter_results.get('retval') == 0 @@ -767,7 +767,7 @@ class JSONResultsReport(ResultsReport): # Get detailed results. detail_results = {} json_results['detailed_results'] = detail_results - for k, v in iter_results.iteritems(): + for k, v in iter_results.items(): if k == 'retval' or k == 'PASS' or k == ['PASS'] or v == 'PASS': continue diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py index dfcce725..e03ea431 100755 --- a/crosperf/results_report_unittest.py +++ b/crosperf/results_report_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2016 The Chromium OS Authors. All rights reserved. @@ -14,8 +14,8 @@ import collections import io import os import unittest +import unittest.mock as mock -import mock import test_flag from benchmark_run import MockBenchmarkRun @@ -83,7 +83,7 @@ def FakePath(ext): def MakeMockExperiment(compiler='gcc'): """Mocks an experiment using the given compiler.""" - mock_experiment_file = io.BytesIO(""" + mock_experiment_file = io.StringIO(""" board: x86-alex remote: 127.0.0.1 locks_dir: /tmp @@ -312,7 +312,7 @@ class JSONResultsReportTest(unittest.TestCase): # Nothing succeeded; we don't send anything more than what's required. required_keys = self._GetRequiredKeys(is_experiment=True) for result in results: - self.assertItemsEqual(result.iterkeys(), required_keys) + self.assertCountEqual(result.keys(), required_keys) def testJSONReportOutputWithSuccesses(self): success_keyvals = { @@ -429,7 +429,7 @@ class PerfReportParserTest(unittest.TestCase): def testParserParsesRealWorldPerfReport(self): report = ParseStandardPerfReport(self._ReadRealPerfReport()) - self.assertItemsEqual(['cycles', 'instructions'], sorted(report.keys())) + self.assertCountEqual(['cycles', 'instructions'], list(report.keys())) # Arbitrarily selected known percentages from the perf report. known_cycles_percentages = { diff --git a/crosperf/schedv2_unittest.py b/crosperf/schedv2_unittest.py index 3ccf6fa4..7b56d723 100755 --- a/crosperf/schedv2_unittest.py +++ b/crosperf/schedv2_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2015 The Chromium OS Authors. All rights reserved. @@ -12,8 +12,7 @@ from __future__ import print_function import functools import io import unittest - -import mock +import unittest.mock as mock import benchmark_run import test_flag @@ -81,7 +80,7 @@ class Schedv2Test(unittest.TestCase): Note - we mock out BenchmarkRun in this step. """ - experiment_file = ExperimentFile(io.BytesIO(expstr)) + experiment_file = ExperimentFile(io.StringIO(expstr)) experiment = ExperimentFactory().GetExperiment( experiment_file, working_directory='', log_dir='') return experiment @@ -200,7 +199,7 @@ class Schedv2Test(unittest.TestCase): # The non-cache-hit brs are put into Schedv2._label_brl_map. self.assertEqual( functools.reduce(lambda a, x: a + len(x[1]), - my_schedv2.get_label_map().iteritems(), 0), 30) + my_schedv2.get_label_map().items(), 0), 30) def test_nocachehit(self): """Test no cache-hit.""" @@ -218,7 +217,7 @@ class Schedv2Test(unittest.TestCase): # The non-cache-hit brs are put into Schedv2._label_brl_map. self.assertEqual( functools.reduce(lambda a, x: a + len(x[1]), - my_schedv2.get_label_map().iteritems(), 0), 60) + my_schedv2.get_label_map().items(), 0), 60) if __name__ == '__main__': diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index a303a065..12a87ae2 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2017 The Chromium OS Authors. All rights reserved. diff --git a/crosperf/settings_unittest.py b/crosperf/settings_unittest.py index e6126923..e127552f 100755 --- a/crosperf/settings_unittest.py +++ b/crosperf/settings_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -9,7 +9,7 @@ from __future__ import print_function import unittest -import mock +import unittest.mock as mock import settings import settings_factory diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index 579e30b2..71ca7e7f 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -21,7 +21,7 @@ TEST_THAT_PATH = '/usr/bin/test_that' # TODO: Need to check whether Skylab is installed and set up correctly. SKYLAB_PATH = '/usr/local/bin/skylab' GS_UTIL = 'src/chromium/depot_tools/gsutil.py' -AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files' +AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files' CHROME_MOUNT_DIR = '/tmp/chrome_root' diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 413822a2..7f3a7bc4 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. @@ -14,7 +14,7 @@ import os.path import time import unittest -import mock +import unittest.mock as mock import suite_runner import label @@ -227,7 +227,7 @@ class SuiteRunnerTest(unittest.TestCase): self.assertEqual( args_list[1], ('/usr/bin/test_that --autotest_dir ' - '~/trunk/src/third_party/autotest/files --fast ' + '/mnt/host/source/src/third_party/autotest/files --fast ' "--board=lumpy --args=' run_local=False test=octane " 'dut_config=\'"\'"\'{"turbostat": true, "top_interval": 3}\'"\'"\' ' 'profiler=custom_perf profiler_args=\'"\'"\'record -a -e ' diff --git a/crosperf/test_cache/compare_output/results.txt b/crosperf/test_cache/compare_output/results.txt Binary files differindex db6803ce..592e7161 100644 --- a/crosperf/test_cache/compare_output/results.txt +++ b/crosperf/test_cache/compare_output/results.txt diff --git a/crosperf/translate_xbuddy.py b/crosperf/translate_xbuddy.py index ca7058ea..80187f9b 100644..100755 --- a/crosperf/translate_xbuddy.py +++ b/crosperf/translate_xbuddy.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/file_lock_machine.py b/file_lock_machine.py index 8493b082..5bba4430 100755 --- a/file_lock_machine.py +++ b/file_lock_machine.py @@ -1,8 +1,12 @@ -#!/usr/bin/env python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Script to lock/unlock machines.""" +from __future__ import division from __future__ import print_function __author__ = 'asharif@google.com (Ahmad Sharif)' @@ -24,8 +28,8 @@ LOCK_SUFFIX = '_check_lock_liveness' # The locks file directory REQUIRES that 'group' only has read/write # privileges and 'world' has no privileges. So the mask must be -# '0027': 0777 - 0027 = 0750. -LOCK_MASK = 0027 +# '0o27': 0o777 - 0o27 = 0o750. +LOCK_MASK = 0o27 def FileCheckName(name): @@ -37,7 +41,8 @@ def OpenLiveCheck(file_name): fd = open(file_name, 'a') try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: + except IOError as e: + logger.GetLogger().LogError(e) raise return fd @@ -80,9 +85,12 @@ class LockDescription(object): def __str__(self): return ' '.join([ - 'Owner: %s' % self.owner, 'Exclusive: %s' % self.exclusive, - 'Counter: %s' % self.counter, 'Time: %s' % self.time, - 'Reason: %s' % self.reason, 'Auto: %s' % self.auto + 'Owner: %s' % self.owner, + 'Exclusive: %s' % self.exclusive, + 'Counter: %s' % self.counter, + 'Time: %s' % self.time, + 'Reason: %s' % self.reason, + 'Auto: %s' % self.auto, ]) @@ -97,6 +105,12 @@ class FileLock(object): self._file = None self._description = None + self.exclusive = None + self.auto = None + self.reason = None + self.time = None + self.owner = None + def getDescription(self): return self._description @@ -118,12 +132,12 @@ class FileLock(object): seconds=int(time.time() - file_lock.getDescription().time)) elapsed_time = '%s ago' % elapsed_time lock_strings.append( - stringify_fmt % - (os.path.basename(file_lock.getFilePath), - file_lock.getDescription().owner, - file_lock.getDescription().exclusive, - file_lock.getDescription().counter, elapsed_time, - file_lock.getDescription().reason, file_lock.getDescription().auto)) + stringify_fmt % (os.path.basename(file_lock.getFilePath), + file_lock.getDescription().owner, + file_lock.getDescription().exclusive, + file_lock.getDescription().counter, elapsed_time, + file_lock.getDescription().reason, + file_lock.getDescription().auto)) table = '\n'.join(lock_strings) return '\n'.join([header, table]) @@ -286,7 +300,7 @@ class Machine(object): sleep = timeout / 10 while True: locked = self.Lock(exclusive, reason) - if locked or not timeout >= 0: + if locked or timeout < 0: break print('Lock not acquired for {0}, wait {1} seconds ...'.format( self._name, sleep)) diff --git a/file_lock_machine_test.py b/file_lock_machine_test.py index 340f9149..bc20a88b 100755 --- a/file_lock_machine_test.py +++ b/file_lock_machine_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. diff --git a/image_chromeos.py b/image_chromeos.py index 726b5659..d5c404dd 100755 --- a/image_chromeos.py +++ b/image_chromeos.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2019 The Chromium OS Authors. All rights reserved. @@ -260,7 +260,7 @@ def DoImage(argv): command = ' '.join(cros_flash_args) # Workaround for crosbug.com/35684. - os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0600) + os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0o600) if log_level == 'average': cmd_executer.SetLogLevel('verbose') @@ -369,7 +369,7 @@ def LocateOrCopyImage(chromeos_root, image, board=None): def GetImageMountCommand(image, rootfs_mp, stateful_mp): image_dir = os.path.dirname(image) image_file = os.path.basename(image) - mount_command = ('cd ~/trunk/src/scripts &&' + mount_command = ('cd /mnt/host/source/src/scripts &&' './mount_gpt_image.sh --from=%s --image=%s' ' --safe --read_only' ' --rootfs_mountpt=%s' @@ -462,7 +462,7 @@ def TryRemountPartitionAsRW(chromeos_root, remote, log_level): ## Safely ignore. l.LogWarning('Failed to remount partition as rw, ' 'probably the image was not built with ' - "\"--noenable_rootfs_verification\", " + '"--noenable_rootfs_verification", ' 'you can safely ignore this.') else: l.LogOutput('Re-mounted partition as writable.') diff --git a/llvm_tools/patch_manager.py b/llvm_tools/patch_manager.py index 82a73ed8..3c83fa96 100755 --- a/llvm_tools/patch_manager.py +++ b/llvm_tools/patch_manager.py @@ -95,6 +95,13 @@ def GetCommandLineArgs(): 'bisecting a patch (default: %(default)s) - only used for ' '"bisect_patches"') + # Trust src_path HEAD and svn_version. + parser.add_argument( + '--use_src_head', + action='store_true', + help='Use the HEAD of src_path directory as is, not necessarily the same ' + 'as the svn_version of upstream.') + # Add argument for the LLVM version to use for patch management. parser.add_argument( '--svn_version', @@ -718,8 +725,9 @@ def main(): # patches that fail to apply could successfully apply if HEAD's SVN version # was the same as 'svn_version'. In other words, HEAD's git hash should be # what is being updated to (e.g. LLVM_NEXT_HASH). - VerifyHEADIsTheSameAsSVNVersion(args_output.src_path, - args_output.svn_version) + if not args_output.use_src_head: + VerifyHEADIsTheSameAsSVNVersion(args_output.src_path, + args_output.svn_version) else: # `git bisect run` called this script. # diff --git a/lock_machine.py b/lock_machine.py index 87230b70..244edfb9 100755 --- a/lock_machine.py +++ b/lock_machine.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2019 The Chromium OS Authors. All rights reserved. @@ -298,7 +298,7 @@ class LockManager(object): cros_machine = cros_machine + '.cros' self.machines = [ - m for m in self.machines if m != cros_machine and m != machine + m for m in self.machines if m not in (cros_machine, machine) ] def CheckMachineLocks(self, machine_states, cmd): @@ -316,7 +316,7 @@ class LockManager(object): Raises: DontOwnLock: The lock on a requested machine is owned by someone else. """ - for k, state in machine_states.iteritems(): + for k, state in machine_states.items(): if cmd == 'unlock': if not state['locked']: self.logger.LogWarning('Attempt to unlock already unlocked machine ' |