Merging 8 commit(s) from Chromium's toolchain-utils

Merged commit digest:
  042613e Add --use_src_head.
  4279700 afdo_metadata: Publish new profiles for kernel 4.4.
  e013418 cros_utils: Do not use encoding in NamedTemporaryFile
  3607744 bisection: add support of kernel
  ff4c61b afdo_metadata: Publish new profiles for kernel 3.18.
  5534af8 crosperf: migration to python 3
  658d779 afdo_metadata: Publish new profiles for kernel 4.4.
  3a13817 afdo_metadata: Publish new profiles for kernel 3.18.

Change-Id: I9c751be432c1bb9f369cb0499f84270cf4215652

48 files changed, 258 insertions, 233 deletions

diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json
index 0cfffeaa..967c3486 100644
--- a/afdo_metadata/kernel_afdo.json
+++ b/afdo_metadata/kernel_afdo.json
@@ -3,9 +3,9 @@
         "name": "R81-12607.58-1578524353"
     }, 
     "chromeos-kernel-4_4": {
-        "name": "R81-12828.0-1579516426"
+        "name": "R81-12861.0-1580726347"
     }, 
     "chromeos-kernel-3_18": {
-        "name": "R81-12828.0-1579516702"
+        "name": "R81-12861.0-1580725953"
     }
 }
\ No newline at end of file
diff --git a/binary_search_tool/bisect_driver.py b/binary_search_tool/bisect_driver.py
index 6a69fbf4..82df15d6 100644
--- a/binary_search_tool/bisect_driver.py
+++ b/binary_search_tool/bisect_driver.py
@@ -5,6 +5,7 @@
 #
 # pylint: disable=not-callable
 # pylint: disable=indentation
+
 """Utilities for bisection of ChromeOS and Android object files.
 
 This module contains a set of utilities to allow bisection between
@@ -25,6 +26,7 @@ import fcntl
 import os
 import shutil
 import subprocess
+import stat
 import sys
 
 VALID_MODES = ('POPULATE_GOOD', 'POPULATE_BAD', 'TRIAGE')
@@ -33,6 +35,8 @@ BAD_CACHE = 'bad'
 LIST_FILE = os.path.join(GOOD_CACHE, '_LIST')
 
 CONTINUE_ON_MISSING = os.environ.get('BISECT_CONTINUE_ON_MISSING', None) == '1'
+CONTINUE_ON_REDUNDANCY = os.environ.get('BISECT_CONTINUE_ON_REDUNDANCY',
+                                        None) == '1'
 WRAPPER_SAFE_MODE = os.environ.get('BISECT_WRAPPER_SAFE_MODE', None) == '1'
 
 
@@ -260,8 +264,13 @@ def cache_file(execargs, bisect_dir, cache, abs_file_path):
         with lock_file(os.path.join(population_dir, '_DUPS'),
                        'a') as dup_object_list:
           dup_object_list.write('%s\n' % abs_file_path)
+        if CONTINUE_ON_REDUNDANCY:
+          return True
         raise Exception(
-            'Trying to cache file %s multiple times.' % abs_file_path)
+            'Trying to cache file %s multiple times. To avoid the error, set \
+            CONTINUE_ON_REDUNDANCY to 1. For reference, the list of such files \
+            will be written to %s' % (abs_file_path,
+                                      os.path.join(population_dir, '_DUPS')))
 
       shutil.copy2(abs_file_path, bisect_path)
       # Set cache object to be read-only so later compilations can't
@@ -289,7 +298,10 @@ def restore_file(bisect_dir, cache, abs_file_path):
   if os.path.exists(cached_path):
     if os.path.exists(abs_file_path):
       os.remove(abs_file_path)
-    os.link(cached_path, abs_file_path)
+    shutil.copy2(cached_path, abs_file_path)
+    # Add write permission to the restored object files as some packages
+    # (such as kernels) may need write permission to delete files.
+    os.chmod(abs_file_path, os.stat(abs_file_path).st_mode | stat.S_IWUSR)
   else:
     raise Error(('%s is missing from %s cache! Unsure how to proceed. Make '
                  'will now crash.' % (cache, cached_path)))
diff --git a/binary_search_tool/common/test_setup.sh b/binary_search_tool/common/test_setup.sh
index c4f5f698..56b8944a 100755
--- a/binary_search_tool/common/test_setup.sh
+++ b/binary_search_tool/common/test_setup.sh
@@ -94,6 +94,10 @@ echo
 
 if [[ "${BISECT_MODE}" == "OBJECT_MODE" ]]; then
   echo "EMERGING ${BISECT_PACKAGE}"
+  echo "sudo rm -rf /build/${BISECT_BOARD}/var/cache/portage/*"
+  sudo rm -rf /build/${BISECT_BOARD}/var/cache/portage/*
+  echo "sudo rm -rf /build/${BISECT_BOARD}/tmp/portage/${BISECT_PACKAGE}*"
+  sudo rm -rf /build/${BISECT_BOARD}/tmp/portage/${BISECT_PACKAGE}*
   CLEAN_DELAY=0 emerge-${BISECT_BOARD} -C ${BISECT_PACKAGE}
   emerge-${BISECT_BOARD} ${BISECT_PACKAGE}
   emerge_status=$?
@@ -105,8 +109,15 @@ if [[ "${BISECT_MODE}" == "OBJECT_MODE" ]]; then
 
   echo
   echo "DEPLOYING"
-  echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}"
-  cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info
+  if [[ ${PACKAGE} == sys-kernel/chromeos-kernel-* ]]; then
+    echo "/mnt/host/source/src/scripts/update_kernel.sh \
+      --remote=${BISECT_REMOTE}"
+    /mnt/host/source/src/scripts/update_kernel.sh --remote=${BISECT_REMOTE}
+  else
+    echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}"
+    cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info
+  fi
+
   deploy_status=$?
 
   if [[ ${deploy_status} -eq 0 ]] ; then
diff --git a/cros_utils/command_executer.py b/cros_utils/command_executer.py
index c4d3fded..39bff5ed 100644..100755
--- a/cros_utils/command_executer.py
+++ b/cros_utils/command_executer.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -17,11 +18,11 @@ import sys
 import tempfile
 import time
 
-import logger
-import misc
+from cros_utils import logger
 
 mock_default = False
 
+CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
 LOG_LEVEL = ('none', 'quiet', 'average', 'verbose')
 
 
@@ -101,6 +102,7 @@ class CommandExecuter(object):
     # In this way the child cannot mess the parent's terminal.
     p = None
     try:
+      # pylint: disable=bad-option-value, subprocess-popen-preexec-fn
       p = subprocess.Popen(
           cmd,
           stdout=subprocess.PIPE,
@@ -238,11 +240,14 @@ class CommandExecuter(object):
     return command
 
   def WriteToTempShFile(self, contents):
-    handle, command_file = tempfile.mkstemp(prefix=os.uname()[1], suffix='.sh')
-    os.write(handle, '#!/bin/bash\n')
-    os.write(handle, contents)
-    os.close(handle)
-    return command_file
+    # TODO(crbug.com/1048938): use encoding='utf-8' when all dependencies have
+    # migrated to python 3.
+    with tempfile.NamedTemporaryFile(
+        'w', delete=False, prefix=os.uname()[1], suffix='.sh') as f:
+      f.write('#!/bin/bash\n')
+      f.write(contents)
+      f.flush()
+    return f.name
 
   def CrosLearnBoard(self, chromeos_root, machine):
     command = self.RemoteAccessInitCommand(chromeos_root, machine)
@@ -362,15 +367,20 @@ class CommandExecuter(object):
     if self.logger:
       self.logger.LogCmd(command, print_to_console=print_to_console)
 
-    handle, command_file = tempfile.mkstemp(
+    # TODO(crbug.com/1048938): use encoding='utf-8' when all dependencies have
+    # migrated to python 3.
+    with tempfile.NamedTemporaryFile(
+        'w',
+        delete=False,
         dir=os.path.join(chromeos_root, 'src/scripts'),
         suffix='.sh',
-        prefix='in_chroot_cmd')
-    os.write(handle, '#!/bin/bash\n')
-    os.write(handle, command)
-    os.write(handle, '\n')
-    os.close(handle)
+        prefix='in_chroot_cmd') as f:
+      f.write('#!/bin/bash\n')
+      f.write(command)
+      f.write('\n')
+      f.flush()
 
+    command_file = f.name
     os.chmod(command_file, 0o777)
 
     # if return_output is set, run a dummy command first to make sure that
@@ -386,7 +396,7 @@ class CommandExecuter(object):
     # Run command_file inside the chroot, making sure that any "~" is expanded
     # by the shell inside the chroot, not outside.
     command = ("cd %s; cros_sdk %s -- bash -c '%s/%s'" %
-               (chromeos_root, cros_sdk_options, misc.CHROMEOS_SCRIPTS_DIR,
+               (chromeos_root, cros_sdk_options, CHROMEOS_SCRIPTS_DIR,
                 os.path.basename(command_file)))
     ret = self.RunCommandGeneric(
         command,
@@ -601,6 +611,7 @@ class CommandExecuter(object):
     # In this way the child cannot mess the parent's terminal.
     pobject = None
     try:
+      # pylint: disable=bad-option-value, subprocess-popen-preexec-fn
       pobject = subprocess.Popen(
           cmd,
           cwd=cwd,
diff --git a/cros_utils/email_sender.py b/cros_utils/email_sender.py
index 3f724f64..0019982e 100755
--- a/cros_utils/email_sender.py
+++ b/cros_utils/email_sender.py
@@ -97,12 +97,14 @@ class EmailSender(object):
 
     if not text_to_send:
       text_to_send = 'Empty message body.'
-    body_fd, body_filename = tempfile.mkstemp()
-    to_be_deleted = [body_filename]
 
+    to_be_deleted = []
     try:
-      os.write(body_fd, text_to_send)
-      os.close(body_fd)
+      with tempfile.NamedTemporaryFile(
+          'w', encoding='utf-8', delete=False) as f:
+        f.write(text_to_send)
+        f.flush()
+      to_be_deleted.append(f.name)
 
       # Fix single-quotes inside the subject. In bash, to escape a single quote
       # (e.g 'don't') you need to replace it with '\'' (e.g. 'don'\''t'). To
@@ -113,11 +115,10 @@ class EmailSender(object):
       if msg_type == 'html':
         command = ("sendgmr --to='%s' --from='%s' --subject='%s' "
                    "--html_file='%s' --body_file=/dev/null" %
-                   (to_list, email_from, subject, body_filename))
+                   (to_list, email_from, subject, f.name))
       else:
-        command = (
-            "sendgmr --to='%s' --from='%s' --subject='%s' "
-            "--body_file='%s'" % (to_list, email_from, subject, body_filename))
+        command = ("sendgmr --to='%s' --from='%s' --subject='%s' "
+                   "--body_file='%s'" % (to_list, email_from, subject, f.name))
 
       if email_cc:
         cc_list = ','.join(email_cc)
@@ -133,10 +134,11 @@ class EmailSender(object):
             report_suffix = '_report.html'
           else:
             report_suffix = '_report.txt'
-          fd, fname = tempfile.mkstemp(suffix=report_suffix)
-          os.write(fd, attachment.content)
-          os.close(fd)
-          attachment_files.append(fname)
+          with tempfile.NamedTemporaryFile(
+              'w', encoding='utf-8', delete=False, suffix=report_suffix) as f:
+            f.write(attachment.content)
+            f.flush()
+          attachment_files.append(f.name)
         files = ','.join(attachment_files)
         command += " --attachment_files='%s'" % files
         to_be_deleted += attachment_files
diff --git a/cros_utils/file_utils.py b/cros_utils/file_utils.py
index fe6fc16f..f0e4064c 100644
--- a/cros_utils/file_utils.py
+++ b/cros_utils/file_utils.py
@@ -10,7 +10,8 @@ from __future__ import print_function
 import errno
 import os
 import shutil
-import command_executer
+
+from cros_utils import command_executer
 
 
 class FileUtils(object):
@@ -65,7 +66,7 @@ class FileUtils(object):
     shutil.rmtree(path, ignore_errors=True)
 
   def WriteFile(self, path, contents):
-    with open(path, 'wb') as f:
+    with open(path, 'w', encoding='utf-8') as f:
       f.write(contents)
 
 
diff --git a/cros_utils/locks.py b/cros_utils/locks.py
index 4ecbe0a9..848e23fc 100644
--- a/cros_utils/locks.py
+++ b/cros_utils/locks.py
@@ -12,7 +12,7 @@ import time
 
 import lock_machine
 
-import logger
+from cros_utils import logger
 
 
 def AcquireLock(machines, chromeos_root, timeout=1200):
diff --git a/cros_utils/misc.py b/cros_utils/misc.py
index 48e3c724..246767f0 100644
--- a/cros_utils/misc.py
+++ b/cros_utils/misc.py
@@ -17,10 +17,10 @@ import shutil
 import sys
 import traceback
 
-import command_executer
-import logger
+from cros_utils import command_executer
+from cros_utils import logger
 
-CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts'
+CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
 TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/'
                         'cros_utils/toolchain_utils.sh')
 
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py
index 9527fde0..300c2d79 100644
--- a/cros_utils/tabulator.py
+++ b/cros_utils/tabulator.py
@@ -72,8 +72,8 @@ import sys
 import numpy
 import scipy
 
-from email_sender import EmailSender
-import misc
+from cros_utils.email_sender import EmailSender
+from cros_utils import misc
 
 
 def _AllFloat(values):
diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py
index f17de1be..74d461a6 100644
--- a/crosperf/benchmark_run.py
+++ b/crosperf/benchmark_run.py
@@ -131,7 +131,7 @@ class BenchmarkRun(threading.Thread):
           self.failure_reason = 'Return value of test suite was non-zero.'
           self.timeline.Record(STATUS_FAILED)
 
-    except Exception, e:
+    except Exception as e:
       self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
       traceback.print_exc()
       if self.timeline.GetLastEvent() != STATUS_FAILED:
diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py
index 5696c107..4cadc35c 100755
--- a/crosperf/benchmark_run_unittest.py
+++ b/crosperf/benchmark_run_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
@@ -9,10 +9,9 @@
 
 from __future__ import print_function
 
-import unittest
 import inspect
-
-import mock
+import unittest
+import unittest.mock as mock
 
 import benchmark_run
 
@@ -121,7 +120,7 @@ class BenchmarkRunTest(unittest.TestCase):
         'machine_manager', 'logger_to_use', 'log_level', 'share_cache',
         'dut_config'
     ]
-    arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
+    arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__)
     self.assertEqual(len(arg_spec.args), len(args_list))
     self.assertEqual(arg_spec.args, args_list)
 
diff --git a/crosperf/benchmark_unittest.py b/crosperf/benchmark_unittest.py
index 63c0a1c0..70508b19 100755
--- a/crosperf/benchmark_unittest.py
+++ b/crosperf/benchmark_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright 2014 The Chromium OS Authors. All rights reserved.
@@ -62,7 +62,7 @@ class BenchmarkTestCase(unittest.TestCase):
         'perf_args', 'suite', 'show_all_results', 'retries', 'run_local',
         'cwp_dso', 'weight'
     ]
-    arg_spec = inspect.getargspec(Benchmark.__init__)
+    arg_spec = inspect.getfullargspec(Benchmark.__init__)
     self.assertEqual(len(arg_spec.args), len(args_list))
     for arg in args_list:
       self.assertIn(arg, arg_spec.args)
diff --git a/crosperf/config_unittest.py b/crosperf/config_unittest.py
index 9e71539d..208f44dc 100755
--- a/crosperf/config_unittest.py
+++ b/crosperf/config_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
diff --git a/crosperf/crosperf b/crosperf/crosperf
index a29dcbfa..c98f2dd4 100755
--- a/crosperf/crosperf
+++ b/crosperf/crosperf
@@ -1,2 +1,7 @@
 #!/bin/bash
-PYTHONPATH=$(dirname $0)/..:$PYTHONPATH exec python $(dirname $0)/crosperf.py "$@"
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+PYTHONPATH=$(dirname "$0")/..:${PYTHONPATH} exec \
+python3 "$(dirname "$0")/crosperf.py" "$@"
diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py
index 75494371..ec07e7c7 100755
--- a/crosperf/crosperf.py
+++ b/crosperf/crosperf.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -110,7 +110,7 @@ def RunCrosperf(argv):
     test_flag.SetTestMode(True)
 
   experiment_file = ExperimentFile(
-      open(experiment_filename, 'rb'), option_settings)
+      open(experiment_filename, encoding='utf-8'), option_settings)
   if not experiment_file.GetGlobalSettings().GetField('name'):
     experiment_name = os.path.basename(experiment_filename)
     experiment_file.GetGlobalSettings().SetField('name', experiment_name)
diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py
index f7ffa1b7..ffd964a2 100755
--- a/crosperf/crosperf_unittest.py
+++ b/crosperf/crosperf_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
@@ -12,11 +12,9 @@ from __future__ import print_function
 
 import argparse
 import io
-import os
 import tempfile
 import unittest
-
-import mock
+import unittest.mock as mock
 
 import crosperf
 import settings_factory
@@ -46,14 +44,14 @@ class CrosperfTest(unittest.TestCase):
   """Crosperf test class."""
 
   def setUp(self):
-    input_file = io.BytesIO(EXPERIMENT_FILE_1)
+    input_file = io.StringIO(EXPERIMENT_FILE_1)
     self.exp_file = experiment_file.ExperimentFile(input_file)
 
   def testDryRun(self):
-    filehandle, filename = tempfile.mkstemp()
-    os.write(filehandle, EXPERIMENT_FILE_1)
-    crosperf.Main(['', filename, '--dry_run'])
-    os.remove(filename)
+    with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f:
+      f.write(EXPERIMENT_FILE_1)
+      f.flush()
+      crosperf.Main(['', f.name, '--dry_run'])
 
   def testConvertOptionsToSettings(self):
     parser = argparse.ArgumentParser()
diff --git a/crosperf/download_images.py b/crosperf/download_images.py
index f855cb60..9bd4a8bf 100644
--- a/crosperf/download_images.py
+++ b/crosperf/download_images.py
@@ -56,8 +56,8 @@ class ImageDownloader(object):
   def GetBuildID(self, chromeos_root, xbuddy_label):
     # Get the translation of the xbuddy_label into the real Google Storage
     # image name.
-    command = ('cd ~/trunk/src/third_party/toolchain-utils/crosperf; '
-               "python translate_xbuddy.py '%s'" % xbuddy_label)
+    command = ('cd /mnt/host/source/src/third_party/toolchain-utils/crosperf; '
+               "./translate_xbuddy.py '%s'" % xbuddy_label)
     _, build_id_tuple_str, _ = self._ce.ChrootRunCommandWOutput(
         chromeos_root, command)
     if not build_id_tuple_str:
@@ -226,7 +226,7 @@ class ImageDownloader(object):
       status = self.VerifyFileExists(chromeos_root, build_id,
                                      autotest_packages_name)
       if status != 0:
-        default_autotest_dir = '~/trunk/src/third_party/autotest/files'
+        default_autotest_dir = '/mnt/host/source/src/third_party/autotest/files'
         print(
             '(Warning: Could not find autotest packages .)\n'
             '(Warning: Defaulting autotest path to %s .' % default_autotest_dir)
diff --git a/crosperf/download_images_buildid_test.py b/crosperf/download_images_buildid_test.py
index a376691c..fc37f2c1 100755
--- a/crosperf/download_images_buildid_test.py
+++ b/crosperf/download_images_buildid_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py
index 94504d3f..62b8d891 100755
--- a/crosperf/download_images_unittest.py
+++ b/crosperf/download_images_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2019 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -10,8 +10,7 @@ from __future__ import print_function
 
 import os
 import unittest
-
-import mock
+import unittest.mock as mock
 
 import download_images
 from cros_utils import command_executer
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index 69e34fcc..f5b17ee2 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
@@ -13,8 +13,7 @@ import io
 import os
 import socket
 import unittest
-
-import mock
+import unittest.mock as mock
 
 from cros_utils import command_executer
 from cros_utils.file_utils import FileUtils
@@ -85,7 +84,7 @@ class ExperimentFactoryTest(unittest.TestCase):
     self.append_benchmark_call_args = []
 
   def testLoadExperimentFile1(self):
-    experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1))
+    experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
     exp = ExperimentFactory().GetExperiment(
         experiment_file, working_directory='', log_dir='')
     self.assertEqual(exp.remote, ['chromeos-alex3'])
@@ -104,7 +103,7 @@ class ExperimentFactoryTest(unittest.TestCase):
     self.assertEqual(exp.labels[0].board, 'x86-alex')
 
   def testLoadExperimentFile2CWP(self):
-    experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_2))
+    experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_2))
     exp = ExperimentFactory().GetExperiment(
         experiment_file, working_directory='', log_dir='')
     self.assertEqual(exp.cwp_dso, 'kallsyms')
@@ -113,7 +112,7 @@ class ExperimentFactoryTest(unittest.TestCase):
     self.assertEqual(exp.benchmarks[1].weight, 0.2)
 
   def testDuplecateBenchmark(self):
-    mock_experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1))
+    mock_experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
     mock_experiment_file.all_settings = []
     benchmark_settings1 = settings_factory.BenchmarkSettings('name')
     mock_experiment_file.all_settings.append(benchmark_settings1)
@@ -125,7 +124,7 @@ class ExperimentFactoryTest(unittest.TestCase):
       ef.GetExperiment(mock_experiment_file, '', '')
 
   def testCWPExceptions(self):
-    mock_experiment_file = ExperimentFile(io.BytesIO(''))
+    mock_experiment_file = ExperimentFile(io.StringIO(''))
     mock_experiment_file.all_settings = []
     global_settings = settings_factory.GlobalSettings('test_name')
     global_settings.SetField('locks_dir', '/tmp')
@@ -305,7 +304,7 @@ class ExperimentFactoryTest(unittest.TestCase):
 
     label_settings.GetXbuddyPath = FakeGetXbuddyPath
 
-    mock_experiment_file = ExperimentFile(io.BytesIO(''))
+    mock_experiment_file = ExperimentFile(io.StringIO(''))
     mock_experiment_file.all_settings = []
 
     test_flag.SetTestMode(True)
@@ -332,7 +331,7 @@ class ExperimentFactoryTest(unittest.TestCase):
 
     # First test. General test.
     exp = ef.GetExperiment(mock_experiment_file, '', '')
-    self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
+    self.assertCountEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
     self.assertEqual(exp.cache_conditions, [0, 2, 1])
     self.assertEqual(exp.log_level, 'average')
 
@@ -354,9 +353,9 @@ class ExperimentFactoryTest(unittest.TestCase):
     test_flag.SetTestMode(True)
     label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros')
     exp = ef.GetExperiment(mock_experiment_file, '', '')
-    self.assertEqual(
+    self.assertCountEqual(
         exp.remote,
-        ['chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80'])
+        ['123.45.67.89', '123.45.76.80', 'chromeos1.cros', 'chromeos2.cros'])
 
     # Third test: Automatic fixing of bad  logging_level param:
     global_settings.SetField('logging_level', 'really loud!')
@@ -392,7 +391,7 @@ class ExperimentFactoryTest(unittest.TestCase):
     self.assertEqual(len(exp.labels), 2)
     self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path')
     self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path')
-    self.assertEqual(
+    self.assertCountEqual(
         exp.remote,
         ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'])
 
@@ -431,7 +430,7 @@ class ExperimentFactoryTest(unittest.TestCase):
       ef.CheckSkylabTool(chromeos_root, log_level)
     self.assertEqual(mock_runcmd.call_count, 1)
     self.assertEqual(
-        err.exception.message, 'Skylab tool not installed '
+        str(err.exception), 'Skylab tool not installed '
         'correctly, please try to manually install it from '
         '/tmp/chromeos/chromeos-admin/lab-tools/setup_lab_tools')
 
diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py
index 33459d11..1d89edad 100644
--- a/crosperf/experiment_file.py
+++ b/crosperf/experiment_file.py
@@ -129,7 +129,7 @@ class ExperimentFile(object):
           self.global_settings.SetField(field[0], field[1], field[2])
         else:
           raise IOError('Unexpected line.')
-    except Exception, err:
+    except Exception as err:
       raise RuntimeError('Line %d: %s\n==> %s' % (reader.LineNo(), str(err),
                                                   reader.CurrentLine(False)))
 
diff --git a/crosperf/experiment_file_unittest.py b/crosperf/experiment_file_unittest.py
index 12b68223..0d4e1e67 100755
--- a/crosperf/experiment_file_unittest.py
+++ b/crosperf/experiment_file_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
@@ -147,13 +147,13 @@ benchmark: PageCycler {
 }
 
 label: image1 {
-\tremote: chromeos-alex3
 \tchromeos_image: /usr/local/google/cros_image1.bin
+\tremote: chromeos-alex3
 }
 
 label: image2 {
-\tremote: chromeos-lumpy1
 \tchromeos_image: /usr/local/google/cros_image2.bin
+\tremote: chromeos-lumpy1
 }\n\n"""
 
 
@@ -161,7 +161,7 @@ class ExperimentFileTest(unittest.TestCase):
   """The main class for Experiment File test."""
 
   def testLoadExperimentFile1(self):
-    input_file = io.BytesIO(EXPERIMENT_FILE_1)
+    input_file = io.StringIO(EXPERIMENT_FILE_1)
     experiment_file = ExperimentFile(input_file)
     global_settings = experiment_file.GetGlobalSettings()
     self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3'])
@@ -181,7 +181,7 @@ class ExperimentFileTest(unittest.TestCase):
     self.assertEqual(label_settings[0].GetField('remote'), ['chromeos-alex3'])
 
   def testOverrideSetting(self):
-    input_file = io.BytesIO(EXPERIMENT_FILE_2)
+    input_file = io.StringIO(EXPERIMENT_FILE_2)
     experiment_file = ExperimentFile(input_file)
     global_settings = experiment_file.GetGlobalSettings()
     self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3'])
@@ -194,11 +194,11 @@ class ExperimentFileTest(unittest.TestCase):
     self.assertEqual(benchmark_settings[1].GetField('iterations'), 2)
 
   def testDuplicateLabel(self):
-    input_file = io.BytesIO(EXPERIMENT_FILE_3)
+    input_file = io.StringIO(EXPERIMENT_FILE_3)
     self.assertRaises(Exception, ExperimentFile, input_file)
 
   def testDuplicateBenchmark(self):
-    input_file = io.BytesIO(EXPERIMENT_FILE_4)
+    input_file = io.StringIO(EXPERIMENT_FILE_4)
     experiment_file = ExperimentFile(input_file)
     benchmark_settings = experiment_file.GetSettings('benchmark')
     self.assertEqual(benchmark_settings[0].name, 'webrtc')
@@ -209,13 +209,13 @@ class ExperimentFileTest(unittest.TestCase):
                      '--story-tag-filter=smoothness')
 
   def testCanonicalize(self):
-    input_file = io.BytesIO(EXPERIMENT_FILE_1)
+    input_file = io.StringIO(EXPERIMENT_FILE_1)
     experiment_file = ExperimentFile(input_file)
     res = experiment_file.Canonicalize()
     self.assertEqual(res, OUTPUT_FILE)
 
   def testLoadDutConfigExperimentFile_Good(self):
-    input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD)
+    input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD)
     experiment_file = ExperimentFile(input_file)
     global_settings = experiment_file.GetGlobalSettings()
     self.assertEqual(global_settings.GetField('turbostat'), False)
@@ -228,22 +228,21 @@ class ExperimentFileTest(unittest.TestCase):
     self.assertEqual(global_settings.GetField('top_interval'), 5)
 
   def testLoadDutConfigExperimentFile_WrongGovernor(self):
-    input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV)
+    input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV)
     with self.assertRaises(RuntimeError) as msg:
       ExperimentFile(input_file)
-    self.assertRegexpMatches(
-        str(msg.exception), 'governor: misspelled_governor')
-    self.assertRegexpMatches(
+    self.assertRegex(str(msg.exception), 'governor: misspelled_governor')
+    self.assertRegex(
         str(msg.exception), "Invalid enum value for field 'governor'."
         r' Must be one of \(performance, powersave, userspace, ondemand,'
         r' conservative, schedutils, sched, interactive\)')
 
   def testLoadDutConfigExperimentFile_WrongCpuUsage(self):
-    input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE)
+    input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE)
     with self.assertRaises(RuntimeError) as msg:
       ExperimentFile(input_file)
-    self.assertRegexpMatches(str(msg.exception), 'cpu_usage: unknown')
-    self.assertRegexpMatches(
+    self.assertRegex(str(msg.exception), 'cpu_usage: unknown')
+    self.assertRegex(
         str(msg.exception), "Invalid enum value for field 'cpu_usage'."
         r' Must be one of \(all, big_only, little_only, exclusive_cores\)')
 
diff --git a/crosperf/experiment_runner_unittest.py b/crosperf/experiment_runner_unittest.py
index 8747de45..4905975c 100755
--- a/crosperf/experiment_runner_unittest.py
+++ b/crosperf/experiment_runner_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
@@ -15,7 +15,7 @@ import os
 import time
 
 import unittest
-import mock
+import unittest.mock as mock
 
 import experiment_runner
 import experiment_status
@@ -109,7 +109,7 @@ class ExperimentRunnerTest(unittest.TestCase):
 
   def make_fake_experiment(self):
     test_flag.SetTestMode(True)
-    experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1))
+    experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
     experiment = ExperimentFactory().GetExperiment(
         experiment_file, working_directory='', log_dir='')
     return experiment
@@ -409,7 +409,7 @@ class ExperimentRunnerTest(unittest.TestCase):
   @mock.patch.object(Result, 'CopyResultsTo')
   @mock.patch.object(Result, 'CleanUp')
   @mock.patch.object(Result, 'FormatStringTop5')
-  @mock.patch('__builtin__.open', new_callable=mock.mock_open)
+  @mock.patch('builtins.open', new_callable=mock.mock_open)
   def test_store_results(self, mock_open, mock_top5, mock_cleanup, mock_copy,
                          _mock_text_report, mock_report, mock_writefile,
                          mock_mkdir, mock_rmdir):
diff --git a/crosperf/flag_test_unittest.py b/crosperf/flag_test_unittest.py
index cb0e59e6..1e77c8a5 100755
--- a/crosperf/flag_test_unittest.py
+++ b/crosperf/flag_test_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
diff --git a/crosperf/generate_report.py b/crosperf/generate_report.py
index cea95a2e..bae365dc 100755
--- a/crosperf/generate_report.py
+++ b/crosperf/generate_report.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2016 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -104,7 +104,7 @@ def CutResultsInPlace(results, max_keys=50, complain_on_update=True):
   }
   """
   actually_updated = False
-  for bench_results in results.itervalues():
+  for bench_results in results.values():
     for platform_results in bench_results:
       for i, result in enumerate(platform_results):
         # Keep the keys that come earliest when sorted alphabetically.
@@ -127,26 +127,6 @@ def CutResultsInPlace(results, max_keys=50, complain_on_update=True):
   return results
 
 
-def _ConvertToASCII(obj):
-  """Convert an object loaded from JSON to ASCII; JSON gives us unicode."""
-
-  # In python 3, we can directly return the unicode loaded from json.
-  if sys.version_info.major == 3:
-    return obj
-  # TODO(zhizhouy): Drop the following code once migrated to python 3.
-  # Using something like `object_hook` is insufficient, since it only fires on
-  # actual JSON objects. `encoding` fails, too, since the default decoder always
-  # uses unicode() to decode strings.
-  # pylint: disable=unicode-builtin
-  if isinstance(obj, unicode):
-    return str(obj)
-  if isinstance(obj, dict):
-    return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()}
-  if isinstance(obj, list):
-    return [_ConvertToASCII(v) for v in obj]
-  return obj
-
-
 def _PositiveInt(s):
   i = int(s)
   if i < 0:
@@ -279,10 +259,8 @@ def _ParseArgs(argv):
 
 def Main(argv):
   args = _ParseArgs(argv)
-  # JSON likes to load UTF-8; our results reporter *really* doesn't like
-  # UTF-8.
   with PickInputFile(args.input) as in_file:
-    raw_results = _ConvertToASCII(json.load(in_file))
+    raw_results = json.load(in_file)
 
   platform_names = raw_results['platforms']
   results = raw_results['data']
diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py
index 465db29b..e19d4695 100755
--- a/crosperf/generate_report_unittest.py
+++ b/crosperf/generate_report_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2016 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -12,7 +12,7 @@ from __future__ import print_function
 import copy
 import json
 import unittest
-import mock
+import unittest.mock as mock
 
 import generate_report
 import results_report
@@ -49,7 +49,7 @@ class GenerateReportTests(unittest.TestCase):
     }
     results = generate_report.CountBenchmarks(runs)
     expected_results = [('foo', 4), ('bar', 0), ('baz', 3)]
-    self.assertEqual(sorted(expected_results), sorted(results))
+    self.assertCountEqual(expected_results, results)
 
   def testCutResultsInPlace(self):
     bench_data = {
@@ -82,8 +82,8 @@ class GenerateReportTests(unittest.TestCase):
         bench_data, max_keys=max_keys, complain_on_update=False)
     # Cuts should be in-place.
     self.assertIs(results, bench_data)
-    self.assertEqual(
-        sorted(original_bench_data.keys()), sorted(bench_data.keys()))
+    self.assertCountEqual(
+        list(original_bench_data.keys()), list(bench_data.keys()))
     for bench_name, original_runs in original_bench_data.items():
       bench_runs = bench_data[bench_name]
       self.assertEqual(len(original_runs), len(bench_runs))
@@ -135,13 +135,11 @@ class GenerateReportTests(unittest.TestCase):
     self.assertEqual(0, return_code)
     self.assertEqual(mock_run_actions.call_count, 1)
     ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]]
-    self.assertEqual(
-        sorted(ctors),
-        sorted([
-            results_report.JSONResultsReport,
-            results_report.TextResultsReport,
-            results_report.HTMLResultsReport,
-        ]))
+    self.assertEqual(ctors, [
+        results_report.JSONResultsReport,
+        results_report.TextResultsReport,
+        results_report.HTMLResultsReport,
+    ])
 
   @mock.patch('generate_report.RunActions')
   def testMainSelectsHTMLIfNoReportsGiven(self, mock_run_actions):
diff --git a/crosperf/label.py b/crosperf/label.py
index bebc2706..b8122613 100644
--- a/crosperf/label.py
+++ b/crosperf/label.py
@@ -93,7 +93,8 @@ class Label(object):
     if self.image_type == 'local':
       self.checksum = ImageChecksummer().Checksum(self, self.log_level)
     elif self.image_type == 'trybot':
-      self.checksum = hashlib.md5(self.chromeos_image).hexdigest()
+      self.checksum = hashlib.md5(
+          self.chromeos_image.encode('utf-8')).hexdigest()
 
   def _GetImageType(self, chromeos_image):
     image_type = None
diff --git a/crosperf/machine_image_manager_unittest.py b/crosperf/machine_image_manager_unittest.py
index 062c185f..fbbca7b6 100755
--- a/crosperf/machine_image_manager_unittest.py
+++ b/crosperf/machine_image_manager_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2015 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -53,13 +53,6 @@ class MachineImageManagerTester(unittest.TestCase):
       duts.append(MockDut(n))
     return duts
 
-  def print_matrix(self, matrix):
-    # pylint: disable=expression-not-assigned
-    for r in matrix:
-      for v in r:
-        print('{} '.format('.' if v == ' ' else v)),
-      print('')
-
   def create_labels_and_duts_from_pattern(self, pattern):
     labels = []
     duts = []
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index 6ed7e513..7211662c 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -151,9 +151,8 @@ class CrosMachine(object):
 
   def _GetMD5Checksum(self, ss):
     if ss:
-      return hashlib.md5(ss).hexdigest()
-    else:
-      return ''
+      return hashlib.md5(ss.encode('utf-8')).hexdigest()
+    return ''
 
   def _GetMachineID(self):
     command = 'dump_vpd_log --full --stdout'
diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py
index f6a77bb8..26eacbd7 100755
--- a/crosperf/machine_manager_unittest.py
+++ b/crosperf/machine_manager_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
@@ -13,8 +13,7 @@ import os.path
 import time
 import hashlib
 import unittest
-
-import mock
+import unittest.mock as mock
 
 import label
 import machine_manager
@@ -778,7 +777,7 @@ class CrosMachineTest(unittest.TestCase):
     self.assertEqual(cm.phys_kbytes, 4194304)
 
     mock_run_cmd.return_value = [1, MEMINFO_STRING, '']
-    self.assertRaises(cm._GetMemoryInfo)
+    self.assertRaises(Exception, cm._GetMemoryInfo)
 
   @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput')
   @mock.patch.object(machine_manager.CrosMachine, 'SetUpChecksumInfo')
@@ -838,7 +837,7 @@ class CrosMachineTest(unittest.TestCase):
         '44:6d:57:20:4a:c5  txqueuelen 1000  (Ethernet)')
 
     mock_run_cmd.return_value = [0, 'invalid hardware config', '']
-    self.assertRaises(cm._GetMachineID)
+    self.assertRaises(Exception, cm._GetMachineID)
 
   def test_add_cooldown_waittime(self):
     cm = machine_manager.CrosMachine('1.2.3.4.cros', '/usr/local/chromeos',
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index bd80e8a7..98062194 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -198,8 +198,8 @@ class Result(object):
       command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir)
       self.ce.RunCommand(command, print_to_console=False)
 
-    command = ('python generate_test_report --no-color --csv %s' %
-               (os.path.join('/tmp', os.path.basename(self.temp_dir))))
+    command = ('./generate_test_report --no-color --csv %s' % (os.path.join(
+        '/tmp', os.path.basename(self.temp_dir))))
     _, out, _ = self.ce.ChrootRunCommandWOutput(
         self.chromeos_root, command, print_to_console=False)
     keyvals_dict = {}
@@ -869,7 +869,7 @@ class Result(object):
     self.suite = suite
     self.cwp_dso = cwp_dso
     # Read in everything from the cache directory.
-    with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
+    with open(os.path.join(cache_dir, RESULTS_FILE), 'rb') as f:
       self.out = pickle.load(f)
       self.err = pickle.load(f)
       self.retval = pickle.load(f)
@@ -907,7 +907,7 @@ class Result(object):
     temp_dir = tempfile.mkdtemp()
 
     # Store to the temp directory.
-    with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f:
+    with open(os.path.join(temp_dir, RESULTS_FILE), 'wb') as f:
       pickle.dump(self.out, f)
       pickle.dump(self.err, f)
       pickle.dump(self.retval, f)
@@ -1040,7 +1040,7 @@ class TelemetryResult(Result):
     self.test_name = test
     self.suite = suite
     self.cwp_dso = cwp_dso
-    with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
+    with open(os.path.join(cache_dir, RESULTS_FILE), 'rb') as f:
       self.out = pickle.load(f)
       self.err = pickle.load(f)
       self.retval = pickle.load(f)
@@ -1183,7 +1183,8 @@ class ResultsCache(object):
     if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
       checksum = '*'
     elif self.label.image_type == 'trybot':
-      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
+      checksum = hashlib.md5(
+          self.label.chromeos_image.encode('utf-8')).hexdigest()
     elif self.label.image_type == 'official':
       checksum = '*'
     else:
@@ -1192,7 +1193,8 @@ class ResultsCache(object):
     if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
       image_path_checksum = '*'
     else:
-      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
+      image_path_checksum = hashlib.md5(
+          self.chromeos_image.encode('utf-8')).hexdigest()
 
     machine_id_checksum = ''
     if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
@@ -1208,7 +1210,7 @@ class ResultsCache(object):
 
     temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args,
                                    self.run_local)
-    test_args_checksum = hashlib.md5(temp_test_args).hexdigest()
+    test_args_checksum = hashlib.md5(temp_test_args.encode('utf-8')).hexdigest()
     return (image_path_checksum, self.test_name, str(self.iteration),
             test_args_checksum, checksum, machine_checksum, machine_id_checksum,
             str(self.CACHE_VERSION))
diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py
index c1c27265..1e7f04a1 100755
--- a/crosperf/results_cache_unittest.py
+++ b/crosperf/results_cache_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
@@ -13,8 +13,7 @@ import os
 import shutil
 import tempfile
 import unittest
-
-import mock
+import unittest.mock as mock
 
 import image_checksummer
 import machine_manager
@@ -236,13 +235,13 @@ TOP_DATA = [
         'top5': [2.0],
     },
     {
-        'cmd': 'sshd',
+        'cmd': 'spi5',
         'cpu_avg': 0.5,
         'count': 1,
         'top5': [1.0],
     },
     {
-        'cmd': 'spi5',
+        'cmd': 'sshd',
         'cpu_avg': 0.5,
         'count': 1,
         'top5': [1.0],
@@ -689,8 +688,7 @@ class ResultTest(unittest.TestCase):
     self.assertEqual(mock_chrootruncmd.call_count, 1)
     self.assertEqual(
         mock_chrootruncmd.call_args_list[0][0],
-        ('/tmp',
-         ('python generate_test_report --no-color --csv %s') % TMP_DIR1))
+        ('/tmp', ('./generate_test_report --no-color --csv %s') % TMP_DIR1))
     self.assertEqual(mock_getpath.call_count, 1)
     self.assertEqual(mock_mkdtemp.call_count, 1)
     self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']})
@@ -893,7 +891,7 @@ class ResultTest(unittest.TestCase):
   def test_process_turbostat_results_with_valid_data(self):
     """Normal case when log exists and contains valid data."""
     self.result.turbostat_log_file = '/tmp/somelogfile.log'
-    with mock.patch('__builtin__.open',
+    with mock.patch('builtins.open',
                     mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT)) as mo:
       cpustats = self.result.ProcessTurbostatResults()
       # Check that the log got opened and data were read/parsed.
@@ -904,7 +902,7 @@ class ResultTest(unittest.TestCase):
   def test_process_turbostat_results_from_empty_file(self):
     """Error case when log exists but file is empty."""
     self.result.turbostat_log_file = '/tmp/emptylogfile.log'
-    with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+    with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo:
       cpustats = self.result.ProcessTurbostatResults()
       # Check that the log got opened and parsed successfully and empty data
       # returned.
@@ -932,7 +930,7 @@ class ResultTest(unittest.TestCase):
     returned cpustats.
     """
     self.result.cpustats_log_file = '/tmp/somelogfile.log'
-    with mock.patch('__builtin__.open',
+    with mock.patch('builtins.open',
                     mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT)) as mo:
       cpustats = self.result.ProcessCpustatsResults()
       # Check that the log got opened and data were read/parsed.
@@ -949,7 +947,7 @@ class ResultTest(unittest.TestCase):
     returned cpustats.
     """
     self.result.cpustats_log_file = '/tmp/somelogfile.log'
-    with mock.patch('__builtin__.open',
+    with mock.patch('builtins.open',
                     mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT)) as mo:
       cpustats = self.result.ProcessCpustatsResults()
       # Check that the log got opened and data were read/parsed.
@@ -960,7 +958,7 @@ class ResultTest(unittest.TestCase):
   def test_process_cpustats_results_from_empty_file(self):
     """Error case when log exists but file is empty."""
     self.result.cpustats_log_file = '/tmp/emptylogfile.log'
-    with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+    with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo:
       cpustats = self.result.ProcessCpustatsResults()
       # Check that the log got opened and parsed successfully and empty data
       # returned.
@@ -972,8 +970,7 @@ class ResultTest(unittest.TestCase):
     """Process top log with valid data."""
 
     self.result.top_log_file = '/tmp/fakelogfile.log'
-    with mock.patch('__builtin__.open',
-                    mock.mock_open(read_data=TOP_LOG)) as mo:
+    with mock.patch('builtins.open', mock.mock_open(read_data=TOP_LOG)) as mo:
       topproc = self.result.ProcessTopResults()
       # Check that the log got opened and data were read/parsed.
       calls = [mock.call('/tmp/fakelogfile.log')]
@@ -983,7 +980,7 @@ class ResultTest(unittest.TestCase):
   def test_process_top_results_from_empty_file(self):
     """Error case when log exists but file is empty."""
     self.result.top_log_file = '/tmp/emptylogfile.log'
-    with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+    with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo:
       topcalls = self.result.ProcessTopResults()
       # Check that the log got opened and parsed successfully and empty data
       # returned.
diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py
index 4879caeb..674745fb 100644
--- a/crosperf/results_organizer.py
+++ b/crosperf/results_organizer.py
@@ -80,7 +80,7 @@ def _GetNonDupLabel(max_dup, runs):
     # pylint: disable=cell-var-from-loop
     added_runs = _Repeat(
         lambda: _DictWithReturnValues(run_retval, run_pass_fail), max_dup)
-    for key, value in run.iteritems():
+    for key, value in run.items():
       match = _DUP_KEY_REGEX.match(key)
       if not match:
         new_run[key] = value
@@ -94,7 +94,7 @@ def _GetNonDupLabel(max_dup, runs):
 
 def _DuplicatePass(result, benchmarks):
   """Properly expands keys like `foo{1}` in `result`."""
-  for bench, data in result.iteritems():
+  for bench, data in result.items():
     max_dup = _GetMaxDup(data)
     # If there's nothing to expand, there's nothing to do.
     if not max_dup:
diff --git a/crosperf/results_organizer_unittest.py b/crosperf/results_organizer_unittest.py
index 39a8cce6..f259879d 100755
--- a/crosperf/results_organizer_unittest.py
+++ b/crosperf/results_organizer_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
diff --git a/crosperf/results_report.py b/crosperf/results_report.py
index 81591db7..ff6c4f96 100644
--- a/crosperf/results_report.py
+++ b/crosperf/results_report.py
@@ -100,10 +100,10 @@ def _FilterPerfReport(event_threshold, report):
 
   def filter_dict(m):
     return {
-        fn_name: pct for fn_name, pct in m.iteritems() if pct >= event_threshold
+        fn_name: pct for fn_name, pct in m.items() if pct >= event_threshold
     }
 
-  return {event: filter_dict(m) for event, m in report.iteritems()}
+  return {event: filter_dict(m) for event, m in report.items()}
 
 
 class _PerfTable(object):
@@ -186,7 +186,7 @@ def _GetTables(benchmark_results, columns, table_type):
   iter_counts = benchmark_results.iter_counts
   result = benchmark_results.run_keyvals
   tables = []
-  for bench_name, runs in result.iteritems():
+  for bench_name, runs in result.items():
     iterations = iter_counts[bench_name]
     ben_table = _GetResultsTableHeader(bench_name, iterations)
 
@@ -438,7 +438,7 @@ class TextResultsReport(ResultsReport):
 
 def _GetHTMLCharts(label_names, test_results):
   charts = []
-  for item, runs in test_results.iteritems():
+  for item, runs in test_results.items():
     # Fun fact: label_names is actually *entirely* useless as a param, since we
     # never add headers. We still need to pass it anyway.
     table = TableGenerator(runs, label_names).GetTable()
@@ -734,7 +734,7 @@ class JSONResultsReport(ResultsReport):
     label_names = benchmark_results.label_names
     summary_field_defaults = self.summary_field_defaults
     final_results = []
-    for test, test_results in benchmark_results.run_keyvals.iteritems():
+    for test, test_results in benchmark_results.run_keyvals.items():
       for label_name, label_results in zip(label_names, test_results):
         for iter_results in label_results:
           passed = iter_results.get('retval') == 0
@@ -767,7 +767,7 @@ class JSONResultsReport(ResultsReport):
           # Get detailed results.
           detail_results = {}
           json_results['detailed_results'] = detail_results
-          for k, v in iter_results.iteritems():
+          for k, v in iter_results.items():
             if k == 'retval' or k == 'PASS' or k == ['PASS'] or v == 'PASS':
               continue
 
diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py
index dfcce725..e03ea431 100755
--- a/crosperf/results_report_unittest.py
+++ b/crosperf/results_report_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright 2016 The Chromium OS Authors. All rights reserved.
@@ -14,8 +14,8 @@ import collections
 import io
 import os
 import unittest
+import unittest.mock as mock
 
-import mock
 import test_flag
 
 from benchmark_run import MockBenchmarkRun
@@ -83,7 +83,7 @@ def FakePath(ext):
 
 def MakeMockExperiment(compiler='gcc'):
   """Mocks an experiment using the given compiler."""
-  mock_experiment_file = io.BytesIO("""
+  mock_experiment_file = io.StringIO("""
       board: x86-alex
       remote: 127.0.0.1
       locks_dir: /tmp
@@ -312,7 +312,7 @@ class JSONResultsReportTest(unittest.TestCase):
     # Nothing succeeded; we don't send anything more than what's required.
     required_keys = self._GetRequiredKeys(is_experiment=True)
     for result in results:
-      self.assertItemsEqual(result.iterkeys(), required_keys)
+      self.assertCountEqual(result.keys(), required_keys)
 
   def testJSONReportOutputWithSuccesses(self):
     success_keyvals = {
@@ -429,7 +429,7 @@ class PerfReportParserTest(unittest.TestCase):
 
   def testParserParsesRealWorldPerfReport(self):
     report = ParseStandardPerfReport(self._ReadRealPerfReport())
-    self.assertItemsEqual(['cycles', 'instructions'], sorted(report.keys()))
+    self.assertCountEqual(['cycles', 'instructions'], list(report.keys()))
 
     # Arbitrarily selected known percentages from the perf report.
     known_cycles_percentages = {
diff --git a/crosperf/schedv2_unittest.py b/crosperf/schedv2_unittest.py
index 3ccf6fa4..7b56d723 100755
--- a/crosperf/schedv2_unittest.py
+++ b/crosperf/schedv2_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright 2015 The Chromium OS Authors. All rights reserved.
@@ -12,8 +12,7 @@ from __future__ import print_function
 import functools
 import io
 import unittest
-
-import mock
+import unittest.mock as mock
 
 import benchmark_run
 import test_flag
@@ -81,7 +80,7 @@ class Schedv2Test(unittest.TestCase):
 
         Note - we mock out BenchmarkRun in this step.
     """
-    experiment_file = ExperimentFile(io.BytesIO(expstr))
+    experiment_file = ExperimentFile(io.StringIO(expstr))
     experiment = ExperimentFactory().GetExperiment(
         experiment_file, working_directory='', log_dir='')
     return experiment
@@ -200,7 +199,7 @@ class Schedv2Test(unittest.TestCase):
       # The non-cache-hit brs are put into Schedv2._label_brl_map.
       self.assertEqual(
           functools.reduce(lambda a, x: a + len(x[1]),
-                           my_schedv2.get_label_map().iteritems(), 0), 30)
+                           my_schedv2.get_label_map().items(), 0), 30)
 
   def test_nocachehit(self):
     """Test no cache-hit."""
@@ -218,7 +217,7 @@ class Schedv2Test(unittest.TestCase):
       # The non-cache-hit brs are put into Schedv2._label_brl_map.
       self.assertEqual(
           functools.reduce(lambda a, x: a + len(x[1]),
-                           my_schedv2.get_label_map().iteritems(), 0), 60)
+                           my_schedv2.get_label_map().items(), 0), 60)
 
 
 if __name__ == '__main__':
diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py
index a303a065..12a87ae2 100755
--- a/crosperf/settings_factory_unittest.py
+++ b/crosperf/settings_factory_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright 2017 The Chromium OS Authors. All rights reserved.
diff --git a/crosperf/settings_unittest.py b/crosperf/settings_unittest.py
index e6126923..e127552f 100755
--- a/crosperf/settings_unittest.py
+++ b/crosperf/settings_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2019 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
@@ -9,7 +9,7 @@
 from __future__ import print_function
 
 import unittest
-import mock
+import unittest.mock as mock
 
 import settings
 import settings_factory
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index 579e30b2..71ca7e7f 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -21,7 +21,7 @@ TEST_THAT_PATH = '/usr/bin/test_that'
 # TODO: Need to check whether Skylab is installed and set up correctly.
 SKYLAB_PATH = '/usr/local/bin/skylab'
 GS_UTIL = 'src/chromium/depot_tools/gsutil.py'
-AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files'
+AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files'
 CHROME_MOUNT_DIR = '/tmp/chrome_root'
 
 
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py
index 413822a2..7f3a7bc4 100755
--- a/crosperf/suite_runner_unittest.py
+++ b/crosperf/suite_runner_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
@@ -14,7 +14,7 @@ import os.path
 import time
 
 import unittest
-import mock
+import unittest.mock as mock
 
 import suite_runner
 import label
@@ -227,7 +227,7 @@ class SuiteRunnerTest(unittest.TestCase):
     self.assertEqual(
         args_list[1],
         ('/usr/bin/test_that --autotest_dir '
-         '~/trunk/src/third_party/autotest/files --fast '
+         '/mnt/host/source/src/third_party/autotest/files --fast '
          "--board=lumpy --args=' run_local=False test=octane "
          'dut_config=\'"\'"\'{"turbostat": true, "top_interval": 3}\'"\'"\' '
          'profiler=custom_perf profiler_args=\'"\'"\'record -a -e '
diff --git a/crosperf/test_cache/compare_output/results.txt b/crosperf/test_cache/compare_output/results.txt
index db6803ce..592e7161 100644
--- a/crosperf/test_cache/compare_output/results.txt
+++ b/crosperf/test_cache/compare_output/results.txt
diff --git a/crosperf/translate_xbuddy.py b/crosperf/translate_xbuddy.py
index ca7058ea..80187f9b 100644..100755
--- a/crosperf/translate_xbuddy.py
+++ b/crosperf/translate_xbuddy.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright 2020 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
diff --git a/file_lock_machine.py b/file_lock_machine.py
index 8493b082..5bba4430 100755
--- a/file_lock_machine.py
+++ b/file_lock_machine.py
@@ -1,8 +1,12 @@
-#!/usr/bin/env python2
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 """Script to lock/unlock machines."""
 
+from __future__ import division
 from __future__ import print_function
 
 __author__ = 'asharif@google.com (Ahmad Sharif)'
@@ -24,8 +28,8 @@ LOCK_SUFFIX = '_check_lock_liveness'
 
 # The locks file directory REQUIRES that 'group' only has read/write
 # privileges and 'world' has no privileges.  So the mask must be
-# '0027': 0777 - 0027 = 0750.
-LOCK_MASK = 0027
+# '0o27': 0o777 - 0o27 = 0o750.
+LOCK_MASK = 0o27
 
 
 def FileCheckName(name):
@@ -37,7 +41,8 @@ def OpenLiveCheck(file_name):
     fd = open(file_name, 'a')
   try:
     fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except IOError:
+  except IOError as e:
+    logger.GetLogger().LogError(e)
     raise
   return fd
 
@@ -80,9 +85,12 @@ class LockDescription(object):
 
   def __str__(self):
     return ' '.join([
-        'Owner: %s' % self.owner, 'Exclusive: %s' % self.exclusive,
-        'Counter: %s' % self.counter, 'Time: %s' % self.time,
-        'Reason: %s' % self.reason, 'Auto: %s' % self.auto
+        'Owner: %s' % self.owner,
+        'Exclusive: %s' % self.exclusive,
+        'Counter: %s' % self.counter,
+        'Time: %s' % self.time,
+        'Reason: %s' % self.reason,
+        'Auto: %s' % self.auto,
     ])
 
 
@@ -97,6 +105,12 @@ class FileLock(object):
     self._file = None
     self._description = None
 
+    self.exclusive = None
+    self.auto = None
+    self.reason = None
+    self.time = None
+    self.owner = None
+
   def getDescription(self):
     return self._description
 
@@ -118,12 +132,12 @@ class FileLock(object):
           seconds=int(time.time() - file_lock.getDescription().time))
       elapsed_time = '%s ago' % elapsed_time
       lock_strings.append(
-          stringify_fmt %
-          (os.path.basename(file_lock.getFilePath),
-           file_lock.getDescription().owner,
-           file_lock.getDescription().exclusive,
-           file_lock.getDescription().counter, elapsed_time,
-           file_lock.getDescription().reason, file_lock.getDescription().auto))
+          stringify_fmt % (os.path.basename(file_lock.getFilePath),
+                           file_lock.getDescription().owner,
+                           file_lock.getDescription().exclusive,
+                           file_lock.getDescription().counter, elapsed_time,
+                           file_lock.getDescription().reason,
+                           file_lock.getDescription().auto))
     table = '\n'.join(lock_strings)
     return '\n'.join([header, table])
 
@@ -286,7 +300,7 @@ class Machine(object):
     sleep = timeout / 10
     while True:
       locked = self.Lock(exclusive, reason)
-      if locked or not timeout >= 0:
+      if locked or timeout < 0:
         break
       print('Lock not acquired for {0}, wait {1} seconds ...'.format(
           self._name, sleep))
diff --git a/file_lock_machine_test.py b/file_lock_machine_test.py
index 340f9149..bc20a88b 100755
--- a/file_lock_machine_test.py
+++ b/file_lock_machine_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright 2019 The Chromium OS Authors. All rights reserved.
diff --git a/image_chromeos.py b/image_chromeos.py
index 726b5659..d5c404dd 100755
--- a/image_chromeos.py
+++ b/image_chromeos.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -260,7 +260,7 @@ def DoImage(argv):
       command = ' '.join(cros_flash_args)
 
       # Workaround for crosbug.com/35684.
-      os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0600)
+      os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0o600)
 
       if log_level == 'average':
         cmd_executer.SetLogLevel('verbose')
@@ -369,7 +369,7 @@ def LocateOrCopyImage(chromeos_root, image, board=None):
 def GetImageMountCommand(image, rootfs_mp, stateful_mp):
   image_dir = os.path.dirname(image)
   image_file = os.path.basename(image)
-  mount_command = ('cd ~/trunk/src/scripts &&'
+  mount_command = ('cd /mnt/host/source/src/scripts &&'
                    './mount_gpt_image.sh --from=%s --image=%s'
                    ' --safe --read_only'
                    ' --rootfs_mountpt=%s'
@@ -462,7 +462,7 @@ def TryRemountPartitionAsRW(chromeos_root, remote, log_level):
     ## Safely ignore.
     l.LogWarning('Failed to remount partition as rw, '
                  'probably the image was not built with '
-                 "\"--noenable_rootfs_verification\", "
+                 '"--noenable_rootfs_verification", '
                  'you can safely ignore this.')
   else:
     l.LogOutput('Re-mounted partition as writable.')
diff --git a/llvm_tools/patch_manager.py b/llvm_tools/patch_manager.py
index 82a73ed8..3c83fa96 100755
--- a/llvm_tools/patch_manager.py
+++ b/llvm_tools/patch_manager.py
@@ -95,6 +95,13 @@ def GetCommandLineArgs():
       'bisecting a patch (default: %(default)s) - only used for '
       '"bisect_patches"')
 
+  # Trust src_path HEAD and svn_version.
+  parser.add_argument(
+      '--use_src_head',
+      action='store_true',
+      help='Use the HEAD of src_path directory as is, not necessarily the same '
+      'as the svn_version of upstream.')
+
   # Add argument for the LLVM version to use for patch management.
   parser.add_argument(
       '--svn_version',
@@ -718,8 +725,9 @@ def main():
     # patches that fail to apply could successfully apply if HEAD's SVN version
     # was the same as 'svn_version'. In other words, HEAD's git hash should be
     # what is being updated to (e.g. LLVM_NEXT_HASH).
-    VerifyHEADIsTheSameAsSVNVersion(args_output.src_path,
-                                    args_output.svn_version)
+    if not args_output.use_src_head:
+      VerifyHEADIsTheSameAsSVNVersion(args_output.src_path,
+                                      args_output.svn_version)
   else:
     # `git bisect run` called this script.
     #
diff --git a/lock_machine.py b/lock_machine.py
index 87230b70..244edfb9 100755
--- a/lock_machine.py
+++ b/lock_machine.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -298,7 +298,7 @@ class LockManager(object):
         cros_machine = cros_machine + '.cros'
 
     self.machines = [
-        m for m in self.machines if m != cros_machine and m != machine
+        m for m in self.machines if m not in (cros_machine, machine)
     ]
 
   def CheckMachineLocks(self, machine_states, cmd):
@@ -316,7 +316,7 @@ class LockManager(object):
     Raises:
       DontOwnLock: The lock on a requested machine is owned by someone else.
     """
-    for k, state in machine_states.iteritems():
+    for k, state in machine_states.items():
       if cmd == 'unlock':
         if not state['locked']:
           self.logger.LogWarning('Attempt to unlock already unlocked machine '