diff options
48 files changed, 1571 insertions, 1348 deletions
diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py index e01cb08b..3652b270 100755 --- a/auto_delete_nightly_test_data.py +++ b/auto_delete_nightly_test_data.py @@ -5,15 +5,15 @@ from __future__ import print_function __author__ = 'shenhan@google.com (Han Shen)' +import argparse import datetime -import optparse import os import re import sys -from utils import command_executer -from utils import constants -from utils import misc +from cros_utils import command_executer +from cros_utils import constants +from cros_utils import misc DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') @@ -72,28 +72,27 @@ def CleanDatedDir(dated_dir, dry_run=False): def ProcessArguments(argv): """Process arguments.""" - parser = optparse.OptionParser( + parser = argparse.ArgumentParser( description='Automatically delete nightly test data directories.', usage='auto_delete_nightly_test_data.py options') - parser.add_option('-d', - '--dry_run', - dest='dry_run', - default=False, - action='store_true', - help='Only print command line, do not execute anything.') - parser.add_option('--days_to_preserve', - dest='days_to_preserve', - default=3, - help=('Specify the number of days (not including today), ' - 'test data generated on these days will *NOT* be ' - 'deleted. Defaults to 3.')) - options, _ = parser.parse_args(argv) + parser.add_argument('-d', + '--dry_run', + dest='dry_run', + default=False, + action='store_true', + help='Only print command line, do not execute anything.') + parser.add_argument('--days_to_preserve', + dest='days_to_preserve', + default=3, + help=('Specify the number of days (not including today),' + ' test data generated on these days will *NOT* be ' + 'deleted. Defaults to 3.')) + options = parser.parse_args(argv) return options def CleanChromeOsTmpAndImages(): """Delete temporaries, images under crostc/chromeos.""" - chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', 'chroot', 'tmp') @@ -158,5 +157,5 @@ def Main(argv): if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/build_chrome_browser.py b/build_chrome_browser.py index 4bec27c2..8effa19d 100755 --- a/build_chrome_browser.py +++ b/build_chrome_browser.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to checkout the ChromeOS source. @@ -7,101 +7,98 @@ This script sets up the ChromeOS source in the given directory, matching a particular release of ChromeOS. """ +from __future__ import print_function + __author__ = 'raymes@google.com (Raymes Khoury)' -import optparse +import argparse import os -import shutil import sys -import build_chromeos -from utils import command_executer -from utils import logger -from utils import misc - -cmd_executer = None +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import misc def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) def Main(argv): """Build Chrome browser.""" - # Common initializations - global cmd_executer + cmd_executer = command_executer.GetCommandExecuter() - parser = optparse.OptionParser() - parser.add_option('--chromeos_root', - dest='chromeos_root', - help='Target directory for ChromeOS installation.') - parser.add_option('--version', dest='version') - parser.add_option('--clean', - dest='clean', - default=False, - action='store_true', - help=('Clean the /var/cache/chromeos-chrome/' - 'chrome-src/src/out_$board dir')) - parser.add_option('--env', - dest='env', - default='', - help='Use the following env') - parser.add_option('--ebuild_version', - dest='ebuild_version', - help='Use this ebuild instead of the default one.') - parser.add_option('--cflags', - dest='cflags', - default='', - help='CFLAGS for the ChromeOS packages') - parser.add_option('--cxxflags', - dest='cxxflags', - default='', - help='CXXFLAGS for the ChromeOS packages') - parser.add_option('--ldflags', - dest='ldflags', - default='', - help='LDFLAGS for the ChromeOS packages') - parser.add_option('--board', - dest='board', - help='ChromeOS target board, e.g. x86-generic') - parser.add_option('--no_build_image', - dest='no_build_image', - default=False, - action='store_true', - help=('Skip build image after building browser.' - 'Defaults to False.')) - parser.add_option('--label', - dest='label', - help='Optional label to apply to the ChromeOS image.') - parser.add_option('--build_image_args', - default='', - dest='build_image_args', - help='Optional arguments to build_image.') - parser.add_option('--cros_workon', - dest='cros_workon', - help='Build using external source tree.') - parser.add_option('--dev', - dest='dev', - default=False, - action='store_true', - help=('Build a dev (eg. writable/large) image. ' - 'Defaults to False.')) - parser.add_option('--debug', - dest='debug', - default=False, - action='store_true', - help=('Build chrome browser using debug mode. ' - 'This option implies --dev. Defaults to false.')) - parser.add_option('--verbose', - dest='verbose', - default=False, - action='store_true', - help='Build with verbose information.') - - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + parser.add_argument('--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('--version', dest='version') + parser.add_argument('--clean', + dest='clean', + default=False, + action='store_true', + help=('Clean the /var/cache/chromeos-chrome/' + 'chrome-src/src/out_$board dir')) + parser.add_argument('--env', + dest='env', + default='', + help='Use the following env') + parser.add_argument('--ebuild_version', + dest='ebuild_version', + help='Use this ebuild instead of the default one.') + parser.add_argument('--cflags', + dest='cflags', + default='', + help='CFLAGS for the ChromeOS packages') + parser.add_argument('--cxxflags', + dest='cxxflags', + default='', + help='CXXFLAGS for the ChromeOS packages') + parser.add_argument('--ldflags', + dest='ldflags', + default='', + help='LDFLAGS for the ChromeOS packages') + parser.add_argument('--board', + dest='board', + help='ChromeOS target board, e.g. x86-generic') + parser.add_argument('--no_build_image', + dest='no_build_image', + default=False, + action='store_true', + help=('Skip build image after building browser.' + 'Defaults to False.')) + parser.add_argument('--label', + dest='label', + help='Optional label to apply to the ChromeOS image.') + parser.add_argument('--build_image_args', + default='', + dest='build_image_args', + help='Optional arguments to build_image.') + parser.add_argument('--cros_workon', + dest='cros_workon', + help='Build using external source tree.') + parser.add_argument('--dev', + dest='dev', + default=False, + action='store_true', + help=('Build a dev (eg. writable/large) image. ' + 'Defaults to False.')) + parser.add_argument('--debug', + dest='debug', + default=False, + action='store_true', + help=('Build chrome browser using debug mode. ' + 'This option implies --dev. Defaults to false.')) + parser.add_argument('--verbose', + dest='verbose', + default=False, + action='store_true', + help='Build with verbose information.') + + options = parser.parse_args(argv) if options.chromeos_root is None: Usage(parser, '--chromeos_root must be set') @@ -169,14 +166,14 @@ def Main(argv): cros_workon_keep = (ret[0] != 0) # Emerge the browser - emerge_browser_command = \ - ('CHROME_ORIGIN={0} {1} ' - "CFLAGS=\"$(portageq-{2} envvar CFLAGS) {3}\" " - "LDFLAGS=\"$(portageq-{2} envvar LDFLAGS) {4}\" " - "CXXFLAGS=\"$(portageq-{2} envvar CXXFLAGS) {5}\" " - '{6} emerge-{2} --buildpkg {7}').format( - chrome_origin, chrome_version, options.board, options.cflags, - options.ldflags, options.cxxflags, options.env, ebuild_version) + emerge_browser_command = ('CHROME_ORIGIN={0} {1} ' + 'CFLAGS="$(portageq-{2} envvar CFLAGS) {3}" ' + 'LDFLAGS="$(portageq-{2} envvar LDFLAGS) {4}" ' + 'CXXFLAGS="$(portageq-{2} envvar CXXFLAGS) {5}" ' + '{6} emerge-{2} --buildpkg {7}').format( + chrome_origin, chrome_version, options.board, + options.cflags, options.ldflags, + options.cxxflags, options.env, ebuild_version) cros_sdk_options = '' if options.cros_workon: @@ -194,7 +191,7 @@ def Main(argv): ret = cmd_executer.ChrootRunCommand(options.chromeos_root, command) # cros_workon failed, not a fatal one, just report it. if ret: - print 'cros_workon stop chromeos-chrome failed.' + print('cros_workon stop chromeos-chrome failed.') if options.no_build_image: return ret @@ -224,7 +221,7 @@ def Main(argv): options.chromeos_root, options.board) real_image_dir_path = os.path.realpath(image_dir_path) command = 'ln -sf -T {0} {1}/{2}'.format( - os.path.basename(real_image_dir_path),\ + os.path.basename(real_image_dir_path),\ os.path.dirname(real_image_dir_path),\ options.label) @@ -236,5 +233,5 @@ def Main(argv): if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/build_chromeos.py b/build_chromeos.py index e4f64268..cb68fd00 100755 --- a/build_chromeos.py +++ b/build_chromeos.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to checkout the ChromeOS source. @@ -7,23 +7,24 @@ This script sets up the ChromeOS source in the given directory, matching a particular release of ChromeOS. """ +from __future__ import print_function + __author__ = ('asharif@google.com (Ahmad Sharif) ' 'llozano@google.com (Luis Lozano) ' 'raymes@google.com (Raymes Khoury) ' 'shenhan@google.com (Han Shen)') -import optparse +import argparse import os import sys -import tc_enter_chroot -from utils import command_executer -from utils import logger -from utils import misc +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import misc def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) @@ -33,76 +34,76 @@ def Main(argv): # Common initializations cmd_executer = command_executer.GetCommandExecuter() - parser = optparse.OptionParser() - parser.add_option('--chromeos_root', - dest='chromeos_root', - help='Target directory for ChromeOS installation.') - parser.add_option('--clobber_chroot', - dest='clobber_chroot', - action='store_true', - help='Delete the chroot and start fresh', - default=False) - parser.add_option('--clobber_board', - dest='clobber_board', - action='store_true', - help='Delete the board and start fresh', - default=False) - parser.add_option('--rebuild', - dest='rebuild', - action='store_true', - help='Rebuild all board packages except the toolchain.', - default=False) - parser.add_option('--cflags', - dest='cflags', - default='', - help='CFLAGS for the ChromeOS packages') - parser.add_option('--cxxflags', - dest='cxxflags', - default='', - help='CXXFLAGS for the ChromeOS packages') - parser.add_option('--ldflags', - dest='ldflags', - default='', - help='LDFLAGS for the ChromeOS packages') - parser.add_option('--board', - dest='board', - help='ChromeOS target board, e.g. x86-generic') - parser.add_option('--package', - dest='package', - help='The package needs to be built') - parser.add_option('--label', - dest='label', - help='Optional label symlink to point to build dir.') - parser.add_option('--dev', - dest='dev', - default=False, - action='store_true', - help=('Make the final image in dev mode (eg writable, ' - 'more space on image). Defaults to False.')) - parser.add_option('--debug', - dest='debug', - default=False, - action='store_true', - help=("Optional. Build chrome browser with \"-g -O0\". " - "Notice, this also turns on \'--dev\'. " - 'Defaults to False.')) - parser.add_option('--env', - dest='env', - default='', - help='Env to pass to build_packages.') - parser.add_option('--vanilla', - dest='vanilla', - default=False, - action='store_true', - help='Use default ChromeOS toolchain.') - parser.add_option('--vanilla_image', - dest='vanilla_image', - default=False, - action='store_true', - help=('Use prebuild packages for building the image. ' - 'It also implies the --vanilla option is set.')) - - options = parser.parse_args(argv[1:])[0] + parser = argparse.ArgumentParser() + parser.add_argument('--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('--clobber_chroot', + dest='clobber_chroot', + action='store_true', + help='Delete the chroot and start fresh', + default=False) + parser.add_argument('--clobber_board', + dest='clobber_board', + action='store_true', + help='Delete the board and start fresh', + default=False) + parser.add_argument('--rebuild', + dest='rebuild', + action='store_true', + help='Rebuild all board packages except the toolchain.', + default=False) + parser.add_argument('--cflags', + dest='cflags', + default='', + help='CFLAGS for the ChromeOS packages') + parser.add_argument('--cxxflags', + dest='cxxflags', + default='', + help='CXXFLAGS for the ChromeOS packages') + parser.add_argument('--ldflags', + dest='ldflags', + default='', + help='LDFLAGS for the ChromeOS packages') + parser.add_argument('--board', + dest='board', + help='ChromeOS target board, e.g. x86-generic') + parser.add_argument('--package', + dest='package', + help='The package needs to be built') + parser.add_argument('--label', + dest='label', + help='Optional label symlink to point to build dir.') + parser.add_argument('--dev', + dest='dev', + default=False, + action='store_true', + help=('Make the final image in dev mode (eg writable, ' + 'more space on image). Defaults to False.')) + parser.add_argument('--debug', + dest='debug', + default=False, + action='store_true', + help=("Optional. Build chrome browser with \"-g -O0\". " + "Notice, this also turns on \'--dev\'. " + 'Defaults to False.')) + parser.add_argument('--env', + dest='env', + default='', + help='Env to pass to build_packages.') + parser.add_argument('--vanilla', + dest='vanilla', + default=False, + action='store_true', + help='Use default ChromeOS toolchain.') + parser.add_argument('--vanilla_image', + dest='vanilla_image', + default=False, + action='store_true', + help=('Use prebuild packages for building the image. ' + 'It also implies the --vanilla option is set.')) + + options = parser.parse_args(argv[1:]) if options.chromeos_root is None: Usage(parser, '--chromeos_root must be set') @@ -158,8 +159,6 @@ def Main(argv): if not os.path.isdir(options.chromeos_root + '/chroot/build/' + options.board) or options.clobber_board: # Run build_tc.py from binary package - rootdir = misc.GetRoot(argv[0])[0] - version_number = misc.GetRoot(rootdir)[1] ret = cmd_executer.ChrootRunCommand(options.chromeos_root, misc.GetSetupBoardCommand( options.board, diff --git a/build_tc.py b/build_tc.py index 2df1639d..55fc5b70 100755 --- a/build_tc.py +++ b/build_tc.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,21 +8,24 @@ This script sets up the toolchain if you give it the gcctools directory. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' +import argparse import getpass -import optparse import os import sys import tempfile import tc_enter_chroot -from utils import command_executer -from utils import constants -from utils import misc +from cros_utils import command_executer +from cros_utils import constants +from cros_utils import misc class ToolchainPart(object): + """Class to hold the toolchain pieces.""" def __init__(self, name, @@ -177,86 +180,86 @@ class ToolchainPart(object): def Main(argv): """The main function.""" # Common initializations - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - default='../../', - help=('ChromeOS root checkout directory' - ' uses ../.. if none given.')) - parser.add_option('-g', - '--gcc_dir', - dest='gcc_dir', - help='The directory where gcc resides.') - parser.add_option('--binutils_dir', - dest='binutils_dir', - help='The directory where binutils resides.') - parser.add_option('-x', - '--gdb_dir', - dest='gdb_dir', - help='The directory where gdb resides.') - parser.add_option('-b', - '--board', - dest='board', - default='x86-agz', - help='The target board.') - parser.add_option('-n', - '--noincremental', - dest='noincremental', - default=False, - action='store_true', - help='Use FEATURES=keepwork to do incremental builds.') - parser.add_option('--cflags', - dest='cflags', - default='', - help='Build a compiler with specified CFLAGS') - parser.add_option('--cxxflags', - dest='cxxflags', - default='', - help='Build a compiler with specified CXXFLAGS') - parser.add_option('--cflags_for_target', - dest='cflags_for_target', - default='', - help='Build the target libraries with specified flags') - parser.add_option('--cxxflags_for_target', - dest='cxxflags_for_target', - default='', - help='Build the target libraries with specified flags') - parser.add_option('--ldflags', - dest='ldflags', - default='', - help='Build a compiler with specified LDFLAGS') - parser.add_option('-d', - '--debug', - dest='debug', - default=False, - action='store_true', - help='Build a compiler with -g3 -O0 appended to both' - ' CFLAGS and CXXFLAGS.') - parser.add_option('-m', - '--mount_only', - dest='mount_only', - default=False, - action='store_true', - help='Just mount the tool directories.') - parser.add_option('-u', - '--unmount_only', - dest='unmount_only', - default=False, - action='store_true', - help='Just unmount the tool directories.') - parser.add_option('--extra_use_flags', - dest='extra_use_flags', - default='', - help='Extra flag for USE, to be passed to the ebuild. ' - "('multislot' and 'mounted_<tool>' are always passed.)") - parser.add_option('--gcc_enable_ccache', - dest='gcc_enable_ccache', - default=False, - action='store_true', - help='Enable ccache for the gcc invocations') - - options, _ = parser.parse_args(argv) + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + default='../../', + help=('ChromeOS root checkout directory' + ' uses ../.. if none given.')) + parser.add_argument('-g', + '--gcc_dir', + dest='gcc_dir', + help='The directory where gcc resides.') + parser.add_argument('--binutils_dir', + dest='binutils_dir', + help='The directory where binutils resides.') + parser.add_argument('-x', + '--gdb_dir', + dest='gdb_dir', + help='The directory where gdb resides.') + parser.add_argument('-b', + '--board', + dest='board', + default='x86-alex', + help='The target board.') + parser.add_argument('-n', + '--noincremental', + dest='noincremental', + default=False, + action='store_true', + help='Use FEATURES=keepwork to do incremental builds.') + parser.add_argument('--cflags', + dest='cflags', + default='', + help='Build a compiler with specified CFLAGS') + parser.add_argument('--cxxflags', + dest='cxxflags', + default='', + help='Build a compiler with specified CXXFLAGS') + parser.add_argument('--cflags_for_target', + dest='cflags_for_target', + default='', + help='Build the target libraries with specified flags') + parser.add_argument('--cxxflags_for_target', + dest='cxxflags_for_target', + default='', + help='Build the target libraries with specified flags') + parser.add_argument('--ldflags', + dest='ldflags', + default='', + help='Build a compiler with specified LDFLAGS') + parser.add_argument('-d', + '--debug', + dest='debug', + default=False, + action='store_true', + help='Build a compiler with -g3 -O0 appended to both' + ' CFLAGS and CXXFLAGS.') + parser.add_argument('-m', + '--mount_only', + dest='mount_only', + default=False, + action='store_true', + help='Just mount the tool directories.') + parser.add_argument('-u', + '--unmount_only', + dest='unmount_only', + default=False, + action='store_true', + help='Just unmount the tool directories.') + parser.add_argument('--extra_use_flags', + dest='extra_use_flags', + default='', + help='Extra flag for USE, to be passed to the ebuild. ' + "('multislot' and 'mounted_<tool>' are always passed.)") + parser.add_argument('--gcc_enable_ccache', + dest='gcc_enable_ccache', + default=False, + action='store_true', + help='Enable ccache for the gcc invocations') + + options = parser.parse_args(argv) chromeos_root = misc.CanonicalizePath(options.chromeos_root) if options.gcc_dir: @@ -325,10 +328,10 @@ def Main(argv): else: rv = rv + tp.Build() finally: - print 'Exiting...' + print('Exiting...') return rv if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/build_tool.py b/build_tool.py index 6ff20645..54d83f28 100755 --- a/build_tool.py +++ b/build_tool.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 """Script to bootstrap the chroot using new toolchain. This script allows you to build/install a customized version of gcc/binutils, @@ -8,14 +8,15 @@ This script must be executed outside chroot. Below is some typical usage - -## Build gcc located at /local/gcc/dir and do a bootstrap using the new compiler -## for the chromeos root. The script tries to find a valid chromeos tree all -## the way up from your current working directory. +## Build gcc located at /local/gcc/dir and do a bootstrap using the new +## compiler for the chromeos root. The script tries to find a valid chromeos +## tree all the way up from your current working directory. ./build_tool.py --gcc_dir=/loca/gcc/dir --bootstrap -## Build binutils, using remote branch "mobile_toolchain_v17" and do a bootstrap -## using the new binutils for the chromeos root. The script tries to find a -## valid chromeos tree all the way up from your current working directory. +## Build binutils, using remote branch "mobile_toolchain_v17" and do a +## bootstrap using the new binutils for the chromeos root. The script tries to +## find a valid chromeos tree all the way up from your current working +## directory. ./build_tool.py --binutils_branch=cros/mobile_toolchain_v17 \ --chromeos_root=/chromeos/dir --bootstrap @@ -24,17 +25,19 @@ Below is some typical usage - --chromeos_root=/chromeos/dir --board=daisy """ +from __future__ import print_function + __author__ = 'shenhan@google.com (Han Shen)' -import optparse +import argparse import os import re import sys import repo_to_repo -from utils import command_executer -from utils import logger -from utils import misc +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import misc REPO_PATH_PATTERN = 'src/third_party/{0}' TEMP_BRANCH_NAME = 'internal_testing_branch_no_use' @@ -43,8 +46,7 @@ EBUILD_PATH_PATTERN = 'src/third_party/chromiumos-overlay/sys-devel/{0}' class Bootstrapper(object): - """Class that handles bootstrap process. - """ + """Class that handles bootstrap process.""" def __init__(self, chromeos_root, @@ -106,6 +108,7 @@ class Bootstrapper(object): Args: tool_name: either 'gcc' or 'binutils' tool_dir: the tool source dir to be used + Returns: True if all succeeded False otherwise. """ @@ -137,7 +140,7 @@ class Bootstrapper(object): # 2. Sync sources from user provided tool dir to chromiumos tool git. local_tool_repo = repo_to_repo.FileRepo(tool_dir) chrome_tool_repo = repo_to_repo.GitRepo(chrome_tool_dir, TEMP_BRANCH_NAME) - chrome_tool_repo._root_dir = chrome_tool_dir + chrome_tool_repo.SetRoot(chrome_tool_dir) # Delete all stuff except '.git' before start mapping. self._ce.RunCommand( 'cd {0} && find . -maxdepth 1 -not -name ".git" -not -name "." ' @@ -200,6 +203,7 @@ class Bootstrapper(object): Args: tool_name: either 'gcc' or 'binutils' tool_branch: tool branch to use + Returns: True: if operation succeeds. Otherwise False. """ @@ -256,6 +260,7 @@ class Bootstrapper(object): Args: tool_name: either "gcc" or "binutils". + Returns: A triplet that consisits of whether operation succeeds or not, tool ebuild file full path and tool ebuild file name. @@ -328,6 +333,7 @@ class Bootstrapper(object): Args: chromeos_root: chromeos source tree tool_name: either "gcc" or "binutils" + Returns: True if operation succeds. """ @@ -356,6 +362,7 @@ class Bootstrapper(object): Args: tool_name: either 'gcc' or 'binutils'. + Returns: Absolute git path for the tool. """ @@ -371,6 +378,7 @@ class Bootstrapper(object): tool_branch_githash: githash for tool_branch tool_branch_tree: treeish for the tool branch tool_ebuild_file: tool ebuild file + Returns: True: if operation succeeded. """ @@ -419,6 +427,7 @@ class Bootstrapper(object): Args: tool_name: either "gcc" or "binutils" + Returns: True if operation succeeds. """ @@ -556,84 +565,86 @@ class Bootstrapper(object): def Main(argv): - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - help=('Optional. ChromeOs root dir. ' - 'When not specified, chromeos root will be deduced ' - 'from current working directory.')) - parser.add_option('--gcc_branch', - dest='gcc_branch', - help=('The branch to test against. ' - 'This branch must be a local branch ' - 'inside "src/third_party/gcc". ' - 'Notice, this must not be used with "--gcc_dir".')) - parser.add_option('--binutils_branch', - dest='binutils_branch', - help=('The branch to test against binutils. ' - 'This branch must be a local branch ' - 'inside "src/third_party/binutils". ' - 'Notice, this must not be used with ' - '"--binutils_dir".')) - parser.add_option('-g', - '--gcc_dir', - dest='gcc_dir', - help=('Use a local gcc tree to do bootstrapping. ' - 'Notice, this must not be used with "--gcc_branch".')) - parser.add_option('--binutils_dir', - dest='binutils_dir', - help=('Use a local binutils tree to do bootstrapping. ' - 'Notice, this must not be used with ' - '"--binutils_branch".')) - parser.add_option('--fixperm', - dest='fixperm', - default=False, - action='store_true', - help=('Fix the (notorious) permission error ' - 'while trying to bootstrap the chroot. ' - 'Note this takes an extra 10-15 minutes ' - 'and is only needed once per chromiumos tree.')) - parser.add_option('--setup_tool_ebuild_file_only', - dest='setup_tool_ebuild_file_only', - default=False, - action='store_true', - help=('Setup gcc and/or binutils ebuild file ' - 'to pick up the branch (--gcc/binutils_branch) or ' - 'use gcc and/or binutils source (--gcc/binutils_dir) ' - 'and exit. Keep chroot as is. This should not be ' - 'used with --gcc/binutils_dir/branch options.')) - parser.add_option('--reset_tool_ebuild_file', - dest='reset_tool_ebuild_file', - default=False, - action='store_true', - help=('Reset the modification that is done by this script.' - 'Note, when this script is running, it will modify ' - 'the active gcc/binutils ebuild file. Use this ' - 'option to reset (what this script has done) ' - 'and exit. This should not be used with -- ' - 'gcc/binutils_dir/branch options.')) - parser.add_option('--board', - dest='board', - default=None, - help=('Only build toolchain for specific board(s). ' - 'Use "host" to build for host. ' - 'Use "," to seperate multiple boards. ' - 'This does not perform a chroot bootstrap.')) - parser.add_option('--bootstrap', - dest='bootstrap', - default=False, - action='store_true', - help=('Performs a chroot bootstrap. ' - 'Note, this will *destroy* your current chroot.')) - parser.add_option('--disable-2nd-bootstrap', - dest='disable_2nd_bootstrap', - default=False, - action='store_true', - help=('Disable a second bootstrap ' - '(build of amd64-host stage).')) - - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help=('Optional. ChromeOs root dir. ' + 'When not specified, chromeos root will be deduced' + ' from current working directory.')) + parser.add_argument('--gcc_branch', + dest='gcc_branch', + help=('The branch to test against. ' + 'This branch must be a local branch ' + 'inside "src/third_party/gcc". ' + 'Notice, this must not be used with "--gcc_dir".')) + parser.add_argument('--binutils_branch', + dest='binutils_branch', + help=('The branch to test against binutils. ' + 'This branch must be a local branch ' + 'inside "src/third_party/binutils". ' + 'Notice, this must not be used with ' + '"--binutils_dir".')) + parser.add_argument('-g', + '--gcc_dir', + dest='gcc_dir', + help=('Use a local gcc tree to do bootstrapping. ' + 'Notice, this must not be used with ' + '"--gcc_branch".')) + parser.add_argument('--binutils_dir', + dest='binutils_dir', + help=('Use a local binutils tree to do bootstrapping. ' + 'Notice, this must not be used with ' + '"--binutils_branch".')) + parser.add_argument('--fixperm', + dest='fixperm', + default=False, + action='store_true', + help=('Fix the (notorious) permission error ' + 'while trying to bootstrap the chroot. ' + 'Note this takes an extra 10-15 minutes ' + 'and is only needed once per chromiumos tree.')) + parser.add_argument('--setup_tool_ebuild_file_only', + dest='setup_tool_ebuild_file_only', + default=False, + action='store_true', + help=('Setup gcc and/or binutils ebuild file ' + 'to pick up the branch (--gcc/binutils_branch) or ' + 'use gcc and/or binutils source ' + '(--gcc/binutils_dir) and exit. Keep chroot as is.' + ' This should not be used with ' + '--gcc/binutils_dir/branch options.')) + parser.add_argument('--reset_tool_ebuild_file', + dest='reset_tool_ebuild_file', + default=False, + action='store_true', + help=('Reset the modification that is done by this ' + 'script. Note, when this script is running, it ' + 'will modify the active gcc/binutils ebuild file. ' + 'Use this option to reset (what this script has ' + 'done) and exit. This should not be used with -- ' + 'gcc/binutils_dir/branch options.')) + parser.add_argument('--board', + dest='board', + default=None, + help=('Only build toolchain for specific board(s). ' + 'Use "host" to build for host. ' + 'Use "," to seperate multiple boards. ' + 'This does not perform a chroot bootstrap.')) + parser.add_argument('--bootstrap', + dest='bootstrap', + default=False, + action='store_true', + help=('Performs a chroot bootstrap. ' + 'Note, this will *destroy* your current chroot.')) + parser.add_argument('--disable-2nd-bootstrap', + dest='disable_2nd_bootstrap', + default=False, + action='store_true', + help=('Disable a second bootstrap ' + '(build of amd64-host stage).')) + + options = parser.parse_args(argv) # Trying to deduce chromeos root from current directory. if not options.chromeos_root: logger.GetLogger().LogOutput('Trying to deduce chromeos root ...') @@ -741,5 +752,5 @@ def Main(argv): if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/chromiumos_image_diff.py b/chromiumos_image_diff.py index 7d9b5e34..68791ac5 100755 --- a/chromiumos_image_diff.py +++ b/chromiumos_image_diff.py @@ -1,8 +1,8 @@ -#!/usr/bin/python +#!/usr/bin/python2 """Diff 2 chromiumos images by comparing each elf file. - The script diffs every *ELF* files by dissembling every *executable* section, - which means it is not a FULL elf differ. + The script diffs every *ELF* files by dissembling every *executable* + section, which means it is not a FULL elf differ. A simple usage example - chromiumos_image_diff.py --image1 image-path-1 --image2 image-path-2 @@ -14,6 +14,8 @@ And this script should be executed outside chroot. """ +from __future__ import print_function + __author__ = 'shenhan@google.com (Han Shen)' import argparse @@ -23,9 +25,9 @@ import sys import tempfile import image_chromeos -from utils import command_executer -from utils import logger -from utils import misc +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import misc class CrosImage(object): @@ -39,6 +41,9 @@ class CrosImage(object): self.logger = logger.GetLogger() self.elf_files = [] self.no_unmount = no_unmount + self.unmount_script = '' + self.stateful = '' + self.rootfs = '' def MountImage(self, mount_basename): """Mount/unpack the image.""" @@ -100,7 +105,7 @@ class CrosImage(object): self.stateful = None self.unmount_script = None - return not self.mounted + return not self.mounted def FindElfFiles(self): """Find all elf files for the image. @@ -276,7 +281,7 @@ def Main(argv): result = False image_comparator = None try: - for image_path in [options.image1, options.image2]: + for i, image_path in enumerate([options.image1, options.image2], start=1): image_path = os.path.realpath(image_path) if not os.path.isfile(image_path): logger.getLogger().LogError('"{0}" is not a file.'.format(image_path)) diff --git a/command_executer_timeout_test.py b/command_executer_timeout_test.py index c7c94cbf..ba0207ef 100755 --- a/command_executer_timeout_test.py +++ b/command_executer_timeout_test.py @@ -1,26 +1,27 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. +"""Timeout test for command_executer.""" + +from __future__ import print_function __author__ = 'asharif@google.com (Ahmad Sharif)' -import optparse -import os -import re +import argparse import sys -from utils import command_executer +from cros_utils import command_executer def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) def Main(argv): - parser = optparse.OptionParser() - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + _ = parser.parse_args(argv) command = 'sleep 1000' ce = command_executer.GetCommandExecuter() @@ -29,4 +30,4 @@ def Main(argv): if __name__ == '__main__': - Main(sys.argv) + Main(sys.argv[1:]) diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 24508c9d..02904ad7 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -69,9 +69,9 @@ class ExperimentFactory(object): of experiments could be produced. """ - def _AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args, - iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local): + def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args, + iterations, rm_chroot_tmp, perf_args, suite, + show_all_results, retries, run_local): """Add all the tests in a set to the benchmarks list.""" for test_name in benchmark_list: telemetry_benchmark = Benchmark( @@ -146,20 +146,20 @@ class ExperimentFactory(object): if suite == 'telemetry_Crosperf': if test_name == 'all_perfv2': - self._AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, - test_args, iterations, rm_chroot_tmp, - perf_args, suite, show_all_results, retries, - run_local) + self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local) elif test_name == 'all_pagecyclers': - self._AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests, - test_args, iterations, rm_chroot_tmp, - perf_args, suite, show_all_results, retries, - run_local) + self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local) elif test_name == 'all_toolchain_perf': - self._AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests, - test_args, iterations, rm_chroot_tmp, - perf_args, suite, show_all_results, retries, - run_local) + self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local) # Add non-telemetry toolchain-perf benchmarks: benchmarks.append(Benchmark('graphics_WebGLAquarium', 'graphics_WebGLAquarium', @@ -172,7 +172,7 @@ class ExperimentFactory(object): retries, run_local=False)) elif test_name == 'all_toolchain_perf_old': - self._AppendBenchmarkSet( + self.AppendBenchmarkSet( benchmarks, telemetry_toolchain_old_perf_tests, test_args, iterations, rm_chroot_tmp, perf_args, suite, show_all_results, retries, run_local) diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 97561008..0e836653 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -1,8 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Unit test for experiment_factory.py""" + +from __future__ import print_function import StringIO import socket @@ -15,12 +18,8 @@ from experiment_factory import ExperimentFactory from experiment_file import ExperimentFile import test_flag import benchmark -import label -import experiment import experiment_factory -import machine_manager import settings_factory -import test_flag EXPERIMENT_FILE_1 = """ board: x86-alex @@ -39,47 +38,54 @@ EXPERIMENT_FILE_1 = """ } """ +# pylint: disable=too-many-function-args class ExperimentFactoryTest(unittest.TestCase): + """Class for running experiment factory unittests.""" + + def setUp(self): + self.append_benchmark_call_args = [] def testLoadExperimentFile1(self): experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) - experiment = ExperimentFactory().GetExperiment(experiment_file, - working_directory='', - log_dir='') - self.assertEqual(experiment.remote, ['chromeos-alex3']) - - self.assertEqual(len(experiment.benchmarks), 1) - self.assertEqual(experiment.benchmarks[0].name, 'PageCycler') - self.assertEqual(experiment.benchmarks[0].test_name, 'PageCycler') - self.assertEqual(experiment.benchmarks[0].iterations, 3) - - self.assertEqual(len(experiment.labels), 2) - self.assertEqual(experiment.labels[0].chromeos_image, + exp = ExperimentFactory().GetExperiment(experiment_file, + working_directory='', + log_dir='') + self.assertEqual(exp.remote, ['chromeos-alex3']) + + self.assertEqual(len(exp.benchmarks), 1) + self.assertEqual(exp.benchmarks[0].name, 'PageCycler') + self.assertEqual(exp.benchmarks[0].test_name, 'PageCycler') + self.assertEqual(exp.benchmarks[0].iterations, 3) + + self.assertEqual(len(exp.labels), 2) + self.assertEqual(exp.labels[0].chromeos_image, '/usr/local/google/cros_image1.bin') - self.assertEqual(experiment.labels[0].board, 'x86-alex') + self.assertEqual(exp.labels[0].board, 'x86-alex') def test_append_benchmark_set(self): ef = ExperimentFactory() bench_list = [] - ef._AppendBenchmarkSet(bench_list, - experiment_factory.telemetry_perfv2_tests, '', 1, - False, '', 'telemetry_Crosperf', False, 0, False) + ef.AppendBenchmarkSet(bench_list, + experiment_factory.telemetry_perfv2_tests, '', 1, + False, '', 'telemetry_Crosperf', False, + 0, + False) self.assertEqual( len(bench_list), len(experiment_factory.telemetry_perfv2_tests)) self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) bench_list = [] - ef._AppendBenchmarkSet(bench_list, - experiment_factory.telemetry_pagecycler_tests, '', 1, - False, '', 'telemetry_Crosperf', False, 0, False) + ef.AppendBenchmarkSet(bench_list, + experiment_factory.telemetry_pagecycler_tests, '', 1, + False, '', 'telemetry_Crosperf', False, 0, False) self.assertEqual( len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) bench_list = [] - ef._AppendBenchmarkSet( + ef.AppendBenchmarkSet( bench_list, experiment_factory.telemetry_toolchain_perf_tests, '', 1, False, '', 'telemetry_Crosperf', False, 0, False) self.assertEqual( @@ -87,8 +93,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) @mock.patch.object(socket, 'gethostname') - @mock.patch.object(machine_manager.MachineManager, 'AddMachine') - def test_get_experiment(self, mock_machine_manager, mock_socket): + def test_get_experiment(self, mock_socket): test_flag.SetTestMode(False) self.append_benchmark_call_args = [] @@ -98,16 +103,20 @@ class ExperimentFactoryTest(unittest.TestCase): 'Helper function for test_get_experiment' arg_list = [bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all] - self.append_benchmark_call_args.append(args_list) + self.append_benchmark_call_args.append(arg_list) def FakeGetDefaultRemotes(board): + if not board: + return [] return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'] def FakeGetXbuddyPath(build, board, chroot, log_level): + if not build or not board or not chroot or not log_level: + return '' return 'fake_image_path' ef = ExperimentFactory() - ef._AppendBenchmarkSet = FakeAppendBenchmarkSet + ef.AppendBenchmarkSet = FakeAppendBenchmarkSet ef.GetDefaultRemotes = FakeGetDefaultRemotes label_settings = settings_factory.LabelSettings('image_label') diff --git a/build_benchmarks.py b/deprecated/build_benchmarks.py index 5734eb8d..c10c74d1 100755 --- a/build_benchmarks.py +++ b/deprecated/build_benchmarks.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to build ChromeOS benchmarks @@ -7,7 +7,9 @@ Inputs: chromeos_root toolchain_root board - [chromeos/cpu/<benchname>|chromeos/browser/[pagecycler|sunspider]|chromeos/startup] + [chromeos/cpu/<benchname> | + chromeos/browser/[pagecycler|sunspider] | + chromeos/startup] This script assumes toolchain has already been built in toolchain_root. @@ -25,17 +27,18 @@ Inputs: """ +from __future__ import print_function + __author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' -import optparse +import argparse import os import sys import re import build_chromeos -import tc_enter_chroot -from utils import command_executer -from utils import logger +from cros_utils import command_executer +from cros_utils import logger KNOWN_BENCHMARKS = [ 'chromeos/startup', 'chromeos/browser/pagecycler', @@ -45,18 +48,21 @@ KNOWN_BENCHMARKS = [ # Commands to build CPU benchmarks. -CPU_BUILDCMD_CLEAN = 'cd /usr/local/toolchain_root/third_party/android_bench/v2_0/CLOSED_SOURCE/%s;\ -python ../../scripts/bench.py --toolchain=/usr/bin --action=clean;' +CPU_BUILDCMD_CLEAN = ('cd /usr/local/toolchain_root/third_party/android_bench/' + 'v2_0/CLOSED_SOURCE/%s; python ../../scripts/bench.py ' + '--toolchain=/usr/bin --action=clean;') -CPU_BUILDCMD_BUILD = 'cd /usr/local/toolchain_root/third_party/android_bench/v2_0/CLOSED_SOURCE/%s;\ -python ../../scripts/bench.py --toolchain=/usr/bin --add_cflags=%s --add_ldflags=%s --makeopts=%s --action=build' +CPU_BUILDCMD_BUILD = ('cd /usr/local/toolchain_root/third_party/android_bench/' + 'v2_0/CLOSED_SOURCE/%s; python ../../scripts/bench.py ' + '--toolchain=/usr/bin --add_cflags=%s --add_ldflags=%s ' + '--makeopts=%s --action=build') # Common initializations cmd_executer = command_executer.GetCommandExecuter() def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) @@ -77,9 +83,11 @@ def CreateRunsh(destdir, benchmark): def CreateBinaryCopy(sourcedir, destdir, copy=None): - """Create links in perflab-bin/destdir/* to sourcedir/* for now, instead of copies + """Create links in perflab-bin/destdir/* to sourcedir/*, instead of copies Args: + sourcedir: directory from which to copy. + destdir: directory to which to copy. copy: when none, make soft links to everything under sourcedir, otherwise copy all to destdir. TODO: remove this parameter if it's determined that CopyFiles can use @@ -117,75 +125,78 @@ def Main(argv): """Build ChromeOS.""" # Common initializations - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - help='Target directory for ChromeOS installation.') - parser.add_option('-t', - '--toolchain_root', - dest='toolchain_root', - help='This is obsolete. Do not use.') - parser.add_option('-r', - '--third_party', - dest='third_party', - help='The third_party dir containing android benchmarks.') - parser.add_option('-C', - '--clean', - dest='clean', - action='store_true', - default=False, - help='Clean up build.'), - parser.add_option('-B', - '--build', - dest='build', - action='store_true', - default=False, - help='Build benchmark.'), - parser.add_option('-O', - '--only_copy', - dest='only_copy', - action='store_true', - default=False, - help='Only copy to perflab-bin - no builds.'), - parser.add_option('--workdir', - dest='workdir', - default='.', - help='Work directory for perflab outputs.') - parser.add_option('--clobber_chroot', - dest='clobber_chroot', - action='store_true', - help='Delete the chroot and start fresh', - default=False) - parser.add_option('--clobber_board', - dest='clobber_board', - action='store_true', - help='Delete the board and start fresh', - default=False) - parser.add_option('--cflags', - dest='cflags', - default='', - help='CFLAGS for the ChromeOS packages') - parser.add_option('--cxxflags', - dest='cxxflags', - default='', - help='CXXFLAGS for the ChromeOS packages') - parser.add_option('--ldflags', - dest='ldflags', - default='', - help='LDFLAGS for the ChromeOS packages') - parser.add_option('--makeopts', - dest='makeopts', - default='', - help='Make options for the ChromeOS packages') - parser.add_option('--board', - dest='board', - help='ChromeOS target board, e.g. x86-generic') - - (options, args) = parser.parse_args(argv[1:]) + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('-t', + '--toolchain_root', + dest='toolchain_root', + help='This is obsolete. Do not use.') + parser.add_argument('-r', + '--third_party', + dest='third_party', + help='The third_party dir containing android ' + 'benchmarks.') + parser.add_argument('-C', + '--clean', + dest='clean', + action='store_true', + default=False, + help='Clean up build.') + parser.add_argument('-B', + '--build', + dest='build', + action='store_true', + default=False, + help='Build benchmark.') + parser.add_argument('-O', + '--only_copy', + dest='only_copy', + action='store_true', + default=False, + help='Only copy to perflab-bin - no builds.') + parser.add_argument('--workdir', + dest='workdir', + default='.', + help='Work directory for perflab outputs.') + parser.add_argument('--clobber_chroot', + dest='clobber_chroot', + action='store_true', + help='Delete the chroot and start fresh', + default=False) + parser.add_argument('--clobber_board', + dest='clobber_board', + action='store_true', + help='Delete the board and start fresh', + default=False) + parser.add_argument('--cflags', + dest='cflags', + default='', + help='CFLAGS for the ChromeOS packages') + parser.add_argument('--cxxflags', + dest='cxxflags', + default='', + help='CXXFLAGS for the ChromeOS packages') + parser.add_argument('--ldflags', + dest='ldflags', + default='', + help='LDFLAGS for the ChromeOS packages') + parser.add_argument('--makeopts', + dest='makeopts', + default='', + help='Make options for the ChromeOS packages') + parser.add_argument('--board', + dest='board', + help='ChromeOS target board, e.g. x86-generic') + # Leftover positional arguments + parser.add_argument('args', nargs='+', help='benchmarks') + + options = parser.parse_args(argv[1:]) # validate args - for arg in args: + for arg in options.args: if arg not in KNOWN_BENCHMARKS: logger.GetLogger().LogFatal('Bad benchmark %s specified' % arg) @@ -207,7 +218,7 @@ def Main(argv): else: third_party = '%s/../../../third_party' % os.path.dirname(__file__) third_party = os.path.realpath(third_party) - for arg in args: + for arg in options.args: # CPU benchmarks if re.match('chromeos/cpu', arg): comps = re.split('/', arg) @@ -245,7 +256,8 @@ def Main(argv): return retval elif re.match('chromeos/startup', arg): if options.build: - # Clean for chromeos/browser and chromeos/startup is a Nop since builds are always from scratch. + # Clean for chromeos/browser and chromeos/startup is a Nop + # since builds are always from scratch. build_args = [ os.path.dirname(os.path.abspath(__file__)) + '/build_chromeos.py', '--chromeos_root=' + options.chromeos_root, diff --git a/compare_benchmarks.py b/deprecated/compare_benchmarks.py index 5892aab0..b6a30cad 100755 --- a/compare_benchmarks.py +++ b/deprecated/compare_benchmarks.py @@ -1,11 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to compare ChromeOS benchmarks Inputs: - <perflab-output directory 1 - baseline> - <perflab-output directory 2 - results> + <perflab-output directory 1 - baseline> + <perflab-output directory 2 - results> --csv - comma separated results This script doesn't really know much about benchmarks. It simply looks for @@ -14,41 +14,41 @@ the results and presents it, along with a geometric mean. """ +from __future__ import print_function + __author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' import glob import math -import optparse -import os +import argparse import re import sys -import image_chromeos -import run_tests -from utils import command_executer -from utils import logger +from cros_utils import command_executer -BENCHDIRS = '%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*/results.txt' +BENCHDIRS = ('%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*' + '/results.txt') # Common initializations cmd_executer = command_executer.GetCommandExecuter() def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) -def GetStats(file): +def GetStats(in_file): """Return stats from file""" - f = open(file, 'r') + f = open(in_file, 'r') pairs = [] for l in f: line = l.strip() # Look for match lines like the following: # METRIC isolated TotalTime_ms (down, scalar) trial_run_0: ['1524.4'] - # METRIC isolated isolated_walltime (down, scalar) trial_run_0: ['167.407445192'] + # METRIC isolated isolated_walltime (down, scalar) trial_run_0: \ + # ['167.407445192'] m = re.match(r"METRIC\s+isolated\s+(\S+).*\['(\d+(?:\.\d+)?)'\]", line) if not m: continue @@ -64,9 +64,9 @@ def GetStats(file): def PrintDash(n): tmpstr = '' - for i in range(n): + for _ in range(n): tmpstr += '-' - print tmpstr + print(tmpstr) def PrintHeaderCSV(hdr): @@ -75,7 +75,7 @@ def PrintHeaderCSV(hdr): if tmpstr != '': tmpstr += ',' tmpstr += hdr[i] - print tmpstr + print(tmpstr) def PrintHeader(hdr): @@ -86,7 +86,7 @@ def PrintHeader(hdr): for i in range(len(hdr)): tmpstr += '%15.15s' % hdr[i] - print tmpstr + print(tmpstr) PrintDash(tot_len * 15) @@ -94,22 +94,24 @@ def Main(argv): """Compare Benchmarks.""" # Common initializations - parser = optparse.OptionParser() - parser.add_option('-c', - '--csv', - dest='csv_output', - action='store_true', - default=False, - help='Output in csv form.') + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--csv', + dest='csv_output', + action='store_true', + default=False, + help='Output in csv form.') + parser.add_argument('args', nargs='+', help='positional arguments: ' + '<baseline-output-dir> <results-output-dir>') - (options, args) = parser.parse_args(argv[1:]) + options = parser.parse_args(argv[1:]) # validate args - if len(args) != 2: + if len(options.args) != 2: Usage(parser, 'Needs <baseline output dir> <results output dir>') - base_dir = args[0] - res_dir = args[1] + base_dir = options.args[0] + res_dir = options.args[1] # find res benchmarks that have results resbenches_glob = BENCHDIRS % res_dir @@ -154,18 +156,18 @@ def Main(argv): speedup = (basestats[key] - stats[key]) / basestats[key] speedup = speedup * 100.0 if options.csv_output: - print '%s,%f,%f,%f' % (key, basestats[key], stats[key], speedup) + print('%s,%f,%f,%f' % (key, basestats[key], stats[key], speedup)) else: - print '%15.15s%15.2f%15.2f%14.2f%%' % (key, basestats[key], - stats[key], speedup) + print('%15.15s%15.2f%15.2f%14.2f%%' % (key, basestats[key], + stats[key], speedup)) prod = math.exp(1.0 / count * math.log(prod)) prod = (1.0 - prod) * 100 if options.csv_output: - print '%s,,,%f' % ('Geomean', prod) + print('%s,,,%f' % ('Geomean', prod)) else: - print '%15.15s%15.15s%15.15s%14.2f%%' % ('Geomean', '', '', prod) - print + print('%15.15s%15.15s%15.15s%14.2f%%' % ('Geomean', '', '', prod)) + print('') return 0 diff --git a/repo_to_repo.py b/deprecated/repo_to_repo.py index b0de0484..b3658883 100755 --- a/repo_to_repo.py +++ b/deprecated/repo_to_repo.py @@ -1,11 +1,15 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. +"""Module for transferring files between various types of repositories.""" + +from __future__ import print_function __author__ = 'asharif@google.com (Ahmad Sharif)' +import argparse import datetime -import optparse +import json import os import re import socket @@ -17,6 +21,7 @@ from utils import command_executer from utils import logger from utils import misc +# pylint: disable=anomalous-backslash-in-string def GetCanonicalMappings(mappings): canonical_mappings = [] @@ -41,6 +46,7 @@ def SplitMapping(mapping): class Repo(object): + """Basic repository base class.""" def __init__(self, no_create_tmp_dir=False): self.repo_type = None @@ -87,6 +93,9 @@ class Repo(object): def GetRoot(self): return self._root_dir + def SetRoot(self, directory): + self._root_dir = directory + def CleanupRoot(self): command = 'rm -rf %s' % self._root_dir return self._ce.RunCommand(command) @@ -99,6 +108,7 @@ class Repo(object): # Note - this type of repo is used only for "readonly", in other words, this # only serves as a incoming repo. class FileRepo(Repo): + """Class for file repositories.""" def __init__(self, address, ignores=None): Repo.__init__(self, no_create_tmp_dir=True) @@ -109,6 +119,8 @@ class FileRepo(Repo): self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now()) self.gerrit = None self._root_dir = self.address + if ignores: + self.ignores += ignores def CleanupRoot(self): """Override to prevent deletion.""" @@ -116,6 +128,8 @@ class FileRepo(Repo): class P4Repo(Repo): + """Class for P4 repositories.""" + def __init__(self, address, mappings, revision=None): Repo.__init__(self) @@ -146,6 +160,7 @@ class P4Repo(Repo): class SvnRepo(Repo): + """Class for svn repositories.""" def __init__(self, address, mappings): Repo.__init__(self) @@ -174,6 +189,7 @@ class SvnRepo(Repo): class GitRepo(Repo): + """Class for git repositories.""" def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None): Repo.__init__(self) @@ -279,6 +295,7 @@ class GitRepo(Repo): class RepoReader(object): + """Class for reading repositories.""" def __init__(self, filename): self.filename = filename @@ -288,7 +305,7 @@ class RepoReader(object): def ParseFile(self): with open(self.filename) as f: - self.main_dict = eval(f.read()) + self.main_dict = json.load(f) self.CreateReposFromDict(self.main_dict) return [self.input_repos, self.output_repos] @@ -331,26 +348,27 @@ class RepoReader(object): @logger.HandleUncaughtExceptions def Main(argv): - parser = optparse.OptionParser() - parser.add_option('-i', - '--input_file', - dest='input_file', - help='The input file that contains repo descriptions.') - - parser.add_option('-n', - '--dry_run', - dest='dry_run', - action='store_true', - default=False, - help='Do a dry run of the push.') - - parser.add_option('-F', - '--message_file', - dest='message_file', - default=None, - help='Use contents of the log file as the commit message.') - - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + parser.add_argument('-i', + '--input_file', + dest='input_file', + help='The input file that contains repo descriptions.') + + parser.add_argument('-n', + '--dry_run', + dest='dry_run', + action='store_true', + default=False, + help='Do a dry run of the push.') + + parser.add_argument('-F', + '--message_file', + dest='message_file', + default=None, + help=('Use contents of the log file as the commit ' + 'message.')) + + options = parser.parse_args(argv) if not options.input_file: parser.print_help() return 1 @@ -399,5 +417,5 @@ def Main(argv): if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/deprecated/repo_to_repo_files/binutils-master.json.rtr b/deprecated/repo_to_repo_files/binutils-master.json.rtr new file mode 100644 index 00000000..e9baf804 --- /dev/null +++ b/deprecated/repo_to_repo_files/binutils-master.json.rtr @@ -0,0 +1,22 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/branches/mobile_toolchain_v15_release_branch/gcctools/google_vendor_src_branch/binutils/binutils-2.21/... ..." + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "type": "git", + "gerrit": true, + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/binutils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr b/deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr new file mode 100644 index 00000000..927a9a84 --- /dev/null +++ b/deprecated/repo_to_repo_files/binutils-mobile_toolchain_v16.json.rtr @@ -0,0 +1,23 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/branches/mobile_toolchain_v16_release_branch/gcctools/google_vendor_src_branch/binutils/binutils-2.22/... ..." + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "gerrit": true, + "type": "git", + "branch": "mobile_toolchain_v16_release_branch", + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/binutils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/crosperf.json.rtr b/deprecated/repo_to_repo_files/crosperf.json.rtr new file mode 100644 index 00000000..95c8f624 --- /dev/null +++ b/deprecated/repo_to_repo_files/crosperf.json.rtr @@ -0,0 +1,38 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/gcctools/chromeos/v14/crosperf/... crosperf/...", + "//depot2/gcctools/chromeos/v14/utils/__init__.py utils/__init__.py", + "//depot2/gcctools/chromeos/v14/utils/colortrans.py utils/colortrans.py", + "//depot2/gcctools/chromeos/v14/utils/command_executer.py utils/command_executer.py", + "//depot2/gcctools/chromeos/v14/utils/email_sender.py utils/email_sender.py", + "//depot2/gcctools/chromeos/v14/utils/file_utils.py utils/file_utils.py", + "//depot2/gcctools/chromeos/v14/utils/logger.py utils/logger.py", + "//depot2/gcctools/chromeos/v14/utils/misc.py utils/misc.py", + "//depot2/gcctools/chromeos/v14/utils/misc_test.py utils/misc_test.py", + "//depot2/gcctools/chromeos/v14/utils/stats.py utils/stats.py", + "//depot2/gcctools/chromeos/v14/utils/tabulator.py utils/tabulator.py", + "//depot2/gcctools/chromeos/v14/utils/tabulator_test.py utils/tabulator_test.py", + "//depot2/gcctools/chromeos/v14/utils/timeline.py utils/timeline.py", + "//depot2/gcctools/chromeos/v14/utils/timeline_test.py utils/timeline_test.py", + "//depot2/gcctools/chromeos/v14/image_chromeos.py image_chromeos.py", + "//depot2/gcctools/chromeos/v14/lock_machine.py lock_machine.py", + "//depot2/gcctools/chromeos/v14/README README" + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "type": "git", + "gerrit": true, + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/toolchain-utils.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr b/deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr new file mode 100644 index 00000000..bc4dadf0 --- /dev/null +++ b/deprecated/repo_to_repo_files/gcc-branches_google_4_7.json.rtr @@ -0,0 +1,22 @@ +{ + "input": [ + { + "type": "svn", + "mappings": [ + "branches/google/gcc-4_7-mobile" + ], + "address": "svn://gcc.gnu.org/svn/gcc" + } + ], + "output": [ + { + "type": "git", + "ignores": [ + ".svn", + ".git" + ], + "branch": "gcc.gnu.org/branches/google/gcc-4_7-mobile", + "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr b/deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr new file mode 100644 index 00000000..f34063d2 --- /dev/null +++ b/deprecated/repo_to_repo_files/gcc-branches_google_main.json.rtr @@ -0,0 +1,22 @@ +{ + "input": [ + { + "type": "svn", + "mappings": [ + "branches/google/main" + ], + "address": "svn://gcc.gnu.org/svn/gcc" + } + ], + "output": [ + { + "type": "git", + "branch": "gcc.gnu.org/branches/google/main", + "ignores": [ + ".svn", + ".git" + ], + "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gcc-master.json.rtr b/deprecated/repo_to_repo_files/gcc-master.json.rtr new file mode 100644 index 00000000..692ae7c9 --- /dev/null +++ b/deprecated/repo_to_repo_files/gcc-master.json.rtr @@ -0,0 +1,23 @@ +{ + "input": [ + { + "type": "svn", + "mappings": [ + "branches/google/gcc-4_7-mobile" + ], + "address": "svn://gcc.gnu.org/svn/gcc" + } + ], + "output": [ + { + "gerrit": true, + "type": "git", + "ignores": [ + ".svn", + ".git" + ], + "branch": "master", + "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/gdb-master.json.rtr b/deprecated/repo_to_repo_files/gdb-master.json.rtr new file mode 100644 index 00000000..b67ec6f7 --- /dev/null +++ b/deprecated/repo_to_repo_files/gdb-master.json.rtr @@ -0,0 +1,20 @@ +{ + "input": [ + { + "type": "git", + "branch": "202befe3cb3ad0eec07eaaf3f0706e42926b6bbb", + "address": "git://sourceware.org/git/gdb.git" + } + ], + "output": [ + { + "type": "git", + "ignores": [ + ".svn", + ".git" + ], + "branch": "chromeos_master", + "address": "https://chromium.googlesource.com/chromiumos/third_party/gdb.git" + } + ] +}
\ No newline at end of file diff --git a/deprecated/repo_to_repo_files/toolchain-utils.json.rtr b/deprecated/repo_to_repo_files/toolchain-utils.json.rtr new file mode 100644 index 00000000..1975d6b5 --- /dev/null +++ b/deprecated/repo_to_repo_files/toolchain-utils.json.rtr @@ -0,0 +1,28 @@ +{ + "input": [ + { + "type": "p4", + "mappings": [ + "//depot2/gcctools/chromeos/v14/crosperf/... crosperf/...", + "//depot2/gcctools/chromeos/v14/utils/... utils/...", + "//depot2/gcctools/chromeos/v14/image_chromeos.py image_chromeos.py", + "//depot2/gcctools/chromeos/v14/lock_machine.py lock_machine.py", + "//depot2/gcctools/chromeos/v14/README README", + "//depot2/gcctools/chromeos/v14/.gitignore .gitignore" + ], + "address": "perforce2:2666" + } + ], + "output": [ + { + "type": "git", + "ignores": [ + ".svn", + ".git", + ".pyc", + "logs" + ], + "address": "https://chrome-internal.googlesource.com/chromeos/toolchain-utils.git" + } + ] +}
\ No newline at end of file diff --git a/report_generator.py b/deprecated/report_generator.py index 7dc53651..c94b51c2 100755 --- a/report_generator.py +++ b/deprecated/report_generator.py @@ -1,20 +1,23 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to compare a baseline results file to a new results file.""" +from __future__ import print_function + __author__ = 'raymes@google.com (Raymes Khoury)' import sys -from utils import logger -from utils import html_tools +from cros_utils import logger +from cros_utils import html_tools PASS = 'pass' FAIL = 'fail' NOT_EXECUTED = 'not executed' -class ResultsReport: +class ResultsReport(object): + """Class for holding report results.""" def __init__(self, report, num_tests_executed, num_passes, num_failures, num_regressions): @@ -48,7 +51,7 @@ class ResultsReport: def Usage(): - print 'Usage: %s baseline_results new_results' % sys.argv[0] + print('Usage: %s baseline_results new_results' % sys.argv[0]) sys.exit(1) @@ -135,7 +138,7 @@ def Main(argv): if len(argv) < 2: Usage() - print GenerateResultsReport(argv[1], argv[2])[0] + print(GenerateResultsReport(argv[1], argv[2])[0]) if __name__ == '__main__': diff --git a/run_benchmarks.py b/deprecated/run_benchmarks.py index 75ef48c5..4818750a 100755 --- a/run_benchmarks.py +++ b/deprecated/run_benchmarks.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to run ChromeOS benchmarks @@ -6,7 +6,9 @@ Inputs: chromeos_root board - [chromeos/cpu/<benchname>|chromeos/browser/[pagecycler|sunspider]|chromeos/startup] + [chromeos/cpu/<benchname>| + chromeos/browser/[pagecycler|sunspider]| + chromeos/startup] hostname/IP of Chromeos machine chromeos/cpu/<benchname> @@ -25,17 +27,21 @@ Inputs: """ +from __future__ import print_function + __author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' -import optparse +import argparse import os import re import sys import image_chromeos import run_tests -from utils import command_executer -from utils import logger +from cros_utils import command_executer +from cros_utils import logger + +# pylint: disable=anomalous-backslash-in-string KNOWN_BENCHMARKS = [ 'chromeos/startup', 'chromeos/browser/pagecycler', @@ -57,49 +63,45 @@ cmd_executer = command_executer.GetCommandExecuter() def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) -def RunBrowserBenchmark(chromeos_root, board, bench, workdir, machine): +def RunBrowserBenchmark(chromeos_root, board, bench, machine): """Run browser benchmarks. Args: chromeos_root: ChromeOS src dir board: Board being tested bench: Name of benchmark (chromeos/browser/*) - workdir: Directory containing benchmark directory machine: name of chromeos machine """ benchname = re.split('/', bench)[2] - benchdir = '%s/%s' % (workdir, benchname) benchname = name_map[benchname] - retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) - return retval + ret = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) + return ret -def RunStartupBenchmark(chromeos_root, board, bench, workdir, machine): +def RunStartupBenchmark(chromeos_root, board, machine): """Run browser benchmarks. Args: chromeos_root: ChromeOS src dir board: Board being tested - bench: Name of benchmark (chromeos/browser/*) - workdir: Directory containing benchmark directory machine: name of chromeos machine """ benchname = 'startup' - benchdir = '%s/%s' % (workdir, benchname) benchname = name_map[benchname] - retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) - return retval + ret = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) + return ret def RunCpuBenchmark(chromeos_root, bench, workdir, machine): """Run CPU benchmark. Args: + chromeos_root: ChromeOS src dir bench: Name of benchmark workdir: directory containing benchmark directory machine: name of chromeos machine @@ -115,20 +117,20 @@ def RunCpuBenchmark(chromeos_root, bench, workdir, machine): # Since this has exclusive access to the machine, # we do not worry about duplicates. args = 'rm -rf /tmp/%s' % benchname - retval = cmd_executer.CrosRunCommand(args, - chromeos_root=chromeos_root, - machine=machine) - if retval: - return retval + retv = cmd_executer.CrosRunCommand(args, + chromeos_root=chromeos_root, + machine=machine) + if retv: + return retv # Copy benchmark directory. - retval = cmd_executer.CopyFiles(benchdir, - '/tmp/' + benchname, - chromeos_root=chromeos_root, - dest_machine=machine, - dest_cros=True) - if retval: - return retval + retv = cmd_executer.CopyFiles(benchdir, + '/tmp/' + benchname, + chromeos_root=chromeos_root, + dest_machine=machine, + dest_cros=True) + if retv: + return retv # Parse bench.mk to extract run flags. @@ -149,34 +151,35 @@ def RunCpuBenchmark(chromeos_root, bench, workdir, machine): chromeos_root=chromeos_root, machine=machine) - return retval + return retv def Main(argv): """Build ChromeOS.""" # Common initializations - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - help='Target directory for ChromeOS installation.') - parser.add_option('-m', - '--machine', - dest='machine', - help='The chromeos host machine.') - parser.add_option('--workdir', - dest='workdir', - default='./perflab-bin', - help='Work directory for perflab outputs.') - parser.add_option('--board', - dest='board', - help='ChromeOS target board, e.g. x86-generic') - - (options, args) = parser.parse_args(argv[1:]) + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('-m', + '--machine', + dest='machine', + help='The chromeos host machine.') + parser.add_argument('--workdir', + dest='workdir', + default='./perflab-bin', + help='Work directory for perflab outputs.') + parser.add_argument('--board', + dest='board', + help='ChromeOS target board, e.g. x86-generic') + parser.add_argumen('args', margs='+', help='Benchmarks to run.') + + options = parser.parse_args(argv[1:]) # validate args - for arg in args: + for arg in options.args: if arg not in KNOWN_BENCHMARKS: logger.GetLogger().LogFatal('Bad benchmark %s specified' % arg) @@ -190,17 +193,17 @@ def Main(argv): Usage(parser, '--machine must be set') found_err = 0 - retval = 0 - for arg in args: + retv = 0 + for arg in options.args: # CPU benchmarks comps = re.split('/', arg) if re.match('chromeos/cpu', arg): benchname = comps[2] - print 'RUNNING %s' % benchname - retval = RunCpuBenchmark(options.chromeos_root, arg, options.workdir, - options.machine) + print('RUNNING %s' % benchname) + retv = RunCpuBenchmark(options.chromeos_root, arg, options.workdir, + options.machine) if not found_err: - found_err = retval + found_err = retv elif re.match('chromeos/startup', arg): benchname = comps[1] image_args = [ @@ -213,10 +216,10 @@ def Main(argv): image_chromeos.Main(image_args) logger.GetLogger().LogOutput('Running %s' % arg) - retval = RunStartupBenchmark(options.chromeos_root, options.board, arg, - options.workdir, options.machine) + retv = RunStartupBenchmark(options.chromeos_root, options.board, + options.machine) if not found_err: - found_err = retval + found_err = retv elif re.match('chromeos/browser', arg): benchname = comps[2] image_args = [ @@ -229,10 +232,10 @@ def Main(argv): image_chromeos.Main(image_args) logger.GetLogger().LogOutput('Running %s' % arg) - retval = RunBrowserBenchmark(options.chromeos_root, options.board, arg, - options.workdir, options.machine) + retv = RunBrowserBenchmark(options.chromeos_root, options.board, arg, + options.machine) if not found_err: - found_err = retval + found_err = retv return found_err diff --git a/sheriff_rotation.py b/deprecated/sheriff_rotation.py index 718bdb26..f6873077 100755 --- a/sheriff_rotation.py +++ b/deprecated/sheriff_rotation.py @@ -1,26 +1,33 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. -"""Script to build the ChromeOS toolchain. +"""Script to rotate the weekly team sheriff. -This script sets up the toolchain if you give it the gcctools directory. +This script determines who the next sheriff is, updates the file +appropriately and sends out email notifying the team. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' +import argparse import datetime import os -import optparse import sys -from utils import constants -from utils import email_sender + +from cros_utils import constants +from cros_utils import email_sender class SheriffHandler(object): + """Main class for handling sheriff rotations.""" + SHERIFF_FILE = os.path.join(constants.CROSTC_WORKSPACE, 'sheriffs.txt') SUBJECT = 'You (%s) are the sheriff for the week: %s - %s' BODY = ('Please see instructions here: ' - 'https://sites.google.com/a/google.com/chromeos-toolchain-team-home2/home/sheriff-s-corner/sheriff-duties') + 'https://sites.google.com/a/google.com/chromeos-toolchain-team-home2' + '/home/sheriff-s-corner/sheriff-duties') def GetWeekInfo(self, day=datetime.datetime.today()): """Return week_start, week_end.""" @@ -29,7 +36,6 @@ class SheriffHandler(object): delta_since_epoch = day - epoch abs_days = abs(delta_since_epoch.days) - 2 # To get it to start from Sat. - weeks_since_epoch = abs_days / 7 day_of_week = abs_days % 7 week_begin = day - datetime.timedelta(days=day_of_week) @@ -78,47 +84,47 @@ class SheriffHandler(object): def Main(argv): - parser = optparse.OptionParser() - parser.add_option('-e', - '--email', - dest='email', - action='store_true', - help='Email the sheriff.') - parser.add_option('-r', - '--rotate', - dest='rotate', - help='Print sheriffs after n rotations.') - parser.add_option('-w', - '--write', - dest='write', - action='store_true', - default=False, - help='Wrote rotated contents to the sheriff file.') - - options, _ = parser.parse_args(argv) + parser = argparse.ArgumentParser() + parser.add_argument('-e', + '--email', + dest='email', + action='store_true', + help='Email the sheriff.') + parser.add_argument('-r', + '--rotate', + dest='rotate', + help='Print sheriffs after n rotations.') + parser.add_argument('-w', + '--write', + dest='write', + action='store_true', + default=False, + help='Wrote rotated contents to the sheriff file.') + + options = parser.parse_args(argv) sheriff_handler = SheriffHandler() current_sheriff = sheriff_handler.GetCurrentSheriff() week_start, week_end = sheriff_handler.GetWeekInfo() - print 'Current sheriff: %s (%s - %s)' % (current_sheriff, week_start, - week_end) + print('Current sheriff: %s (%s - %s)' % (current_sheriff, week_start, + week_end)) if options.email: sheriff_handler.Email() if options.rotate: rotated_sheriffs = sheriff_handler.GetRotatedSheriffs(int(options.rotate)) - print 'Rotated sheriffs (after %s rotations)' % options.rotate - print '\n'.join(rotated_sheriffs) + print('Rotated sheriffs (after %s rotations)' % options.rotate) + print('\n'.join(rotated_sheriffs)) if options.write: sheriff_handler.WriteSheriffsAsList(rotated_sheriffs) - print 'Rotated sheriffs written to file.' + print('Rotated sheriffs written to file.') return 0 if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/summarize_results.py b/deprecated/summarize_results.py index 69ab4c25..67d7e9a4 100755 --- a/summarize_results.py +++ b/deprecated/summarize_results.py @@ -1,11 +1,13 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to summarize the results of various log files.""" +from __future__ import print_function + __author__ = 'raymes@google.com (Raymes Khoury)' -from utils import command_executer +from cros_utils import command_executer import os import sys import re @@ -13,8 +15,13 @@ import re RESULTS_DIR = 'results' RESULTS_FILE = RESULTS_DIR + '/results.csv' +# pylint: disable=anomalous-backslash-in-string + +class DejaGNUSummarizer(object): + """DejaGNU Summarizer Class""" -class DejaGNUSummarizer: + def __int__(self): + pass def Matches(self, log_file): for log_line in log_file: @@ -43,7 +50,11 @@ class DejaGNUSummarizer: return result -class PerflabSummarizer: +class PerflabSummarizer(object): + """Perflab Summarizer class""" + + def __init__(self): + pass def Matches(self, log_file): p = re.compile('METRIC isolated \w+') @@ -60,11 +71,15 @@ class PerflabSummarizer: for match in matches: if len(match) != 2: continue - result += '%s\t%s\n' % (match[0], match[1]) + result += '%s\t%s\t%s\n' % (match[0], match[1], filename) return result -class AutoTestSummarizer: +class AutoTestSummarizer(object): + """AutoTest Summarizer class""" + + def __init__(self): + pass def Matches(self, log_file): for log_line in log_file: @@ -90,23 +105,23 @@ class AutoTestSummarizer: def Usage(): - print 'Usage: %s log_file' % sys.argv[0] + print('Usage: %s log_file' % sys.argv[0]) sys.exit(1) def SummarizeFile(filename): summarizers = [DejaGNUSummarizer(), AutoTestSummarizer(), PerflabSummarizer()] - input = open(filename, 'rb') + inp = open(filename, 'rb') executer = command_executer.GetCommandExecuter() for summarizer in summarizers: - input.seek(0) - if summarizer.Matches(input): + inp.seek(0) + if summarizer.Matches(inp): executer.CopyFiles(filename, RESULTS_DIR, recursive=False) - input.seek(0) - result = summarizer.Summarize(input, os.path.basename(filename)) - input.close() + inp.seek(0) + result = summarizer.Summarize(inp, os.path.basename(filename)) + inp.close() return result - input.close() + inp.close() return None diff --git a/file_lock_machine.py b/file_lock_machine.py index 5e5c17ca..9b1d3367 100755 --- a/file_lock_machine.py +++ b/file_lock_machine.py @@ -1,22 +1,24 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to lock/unlock machines.""" +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' +import argparse import datetime import fcntl import getpass import glob import json -import optparse import os import socket import sys import time -from utils import logger +from cros_utils import logger LOCK_SUFFIX = '_check_lock_liveness' @@ -32,7 +34,7 @@ def FileCheckName(name): def OpenLiveCheck(file_name): with FileCreationMask(LOCK_MASK): - fd = open(file_name, 'a+w') + fd = open(file_name, 'a') try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: @@ -41,14 +43,16 @@ def OpenLiveCheck(file_name): class FileCreationMask(object): + """Class for the file creation mask.""" def __init__(self, mask): self._mask = mask + self._old_mask = None def __enter__(self): self._old_mask = os.umask(self._mask) - def __exit__(self, type, value, traceback): + def __exit__(self, typ, value, traceback): os.umask(self._old_mask) @@ -89,6 +93,16 @@ class FileLock(object): lock_dir = os.path.dirname(lock_filename) assert os.path.isdir(lock_dir), ("Locks dir: %s doesn't exist!" % lock_dir) self._file = None + self._description = None + + def getDescription(self): + return self._description + + def getFilePath(self): + return self._filepath + + def setDescription(self, desc): + self._description = desc @classmethod def AsString(cls, file_locks): @@ -99,14 +113,16 @@ class FileLock(object): for file_lock in file_locks: elapsed_time = datetime.timedelta( - seconds=int(time.time() - file_lock._description.time)) + seconds=int(time.time() - file_lock.getDescription().time)) elapsed_time = '%s ago' % elapsed_time lock_strings.append( stringify_fmt % - (os.path.basename(file_lock._filepath), file_lock._description.owner, - file_lock._description.exclusive, file_lock._description.counter, - elapsed_time, file_lock._description.reason, - file_lock._description.auto)) + (os.path.basename(file_lock.getFilePath), + file_lock.getDescription().owner, + file_lock.getDescription().exclusive, + file_lock.getDescription().counter, + elapsed_time, file_lock.getDescription().reason, + file_lock.getDescription().auto)) table = '\n'.join(lock_strings) return '\n'.join([header, table]) @@ -161,7 +177,7 @@ class FileLock(object): logger.GetLogger().LogError(ex) return None - def __exit__(self, type, value, traceback): + def __exit__(self, typ, value, traceback): self._file.truncate(0) self._file.write(json.dumps(self._description.__dict__, skipkeys=True)) self._file.close() @@ -171,6 +187,7 @@ class FileLock(object): class Lock(object): + """Lock class""" def __init__(self, lock_file, auto=True): self._to_lock = os.path.basename(lock_file) @@ -245,6 +262,8 @@ class Lock(object): class Machine(object): + """Machine class""" + LOCKS_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/locks' def __init__(self, name, locks_dir=LOCKS_DIR, auto=True): @@ -267,8 +286,9 @@ class Machine(object): locked = self.Lock(exclusive, reason) if locked or not timeout >= 0: break - print 'Lock not acquired for {0}, wait {1} seconds ...'.format(self._name, - sleep) + print('Lock not acquired for {0}, wait {1} seconds ...'.format( + self._name, + sleep)) time.sleep(sleep) timeout -= sleep return locked @@ -280,55 +300,57 @@ class Machine(object): def Main(argv): """The main function.""" - parser = optparse.OptionParser() - parser.add_option('-r', - '--reason', - dest='reason', - default='', - help='The lock reason.') - parser.add_option('-u', - '--unlock', - dest='unlock', - action='store_true', - default=False, - help='Use this to unlock.') - parser.add_option('-l', - '--list_locks', - dest='list_locks', - action='store_true', - default=False, - help='Use this to list locks.') - parser.add_option('-f', - '--ignore_ownership', - dest='ignore_ownership', - action='store_true', - default=False, - help="Use this to force unlock on a lock you don't own.") - parser.add_option('-s', - '--shared', - dest='shared', - action='store_true', - default=False, - help='Use this for a shared (non-exclusive) lock.') - parser.add_option('-d', - '--dir', - dest='locks_dir', - action='store', - default=Machine.LOCKS_DIR, - help='Use this to set different locks_dir') - - options, args = parser.parse_args(argv) + + parser = argparse.ArgumentParser() + parser.add_argument('-r', + '--reason', + dest='reason', + default='', + help='The lock reason.') + parser.add_argument('-u', + '--unlock', + dest='unlock', + action='store_true', + default=False, + help='Use this to unlock.') + parser.add_argument('-l', + '--list_locks', + dest='list_locks', + action='store_true', + default=False, + help='Use this to list locks.') + parser.add_argument('-f', + '--ignore_ownership', + dest='ignore_ownership', + action='store_true', + default=False, + help="Use this to force unlock on a lock you don't own.") + parser.add_argument('-s', + '--shared', + dest='shared', + action='store_true', + default=False, + help='Use this for a shared (non-exclusive) lock.') + parser.add_argument('-d', + '--dir', + dest='locks_dir', + action='store', + default=Machine.LOCKS_DIR, + help='Use this to set different locks_dir') + parser.add_argument('args', nargs='*', help='Machine arg.') + + options = parser.parse_args(argv) options.locks_dir = os.path.abspath(options.locks_dir) exclusive = not options.shared - if not options.list_locks and len(args) != 2: + if not options.list_locks and len(options.args) != 2: logger.GetLogger().LogError( 'Either --list_locks or a machine arg is needed.') return 1 - if len(args) > 1: - machine = Machine(args[1], options.locks_dir, auto=False) + if len(options.args) > 1: + machine = Machine(options.args[1], options.locks_dir, auto=False) else: machine = None @@ -347,4 +369,4 @@ def Main(argv): if __name__ == '__main__': - sys.exit(Main(sys.argv)) + sys.exit(Main(sys.argv[1:])) diff --git a/get_common_image_version.py b/get_common_image_version.py index bf5d219b..da36b98f 100755 --- a/get_common_image_version.py +++ b/get_common_image_version.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2013 Google Inc. All Rights Reserved. """Script to find list of common images (first beta releases) in Chromeos. @@ -10,9 +10,11 @@ using randomly selected versions. Currently we define as a "stable" version the first Beta release in a particular release cycle. """ +from __future__ import print_function + __author__ = 'llozano@google.com (Luis Lozano)' -import optparse +import argparse import pickle import re import sys @@ -22,9 +24,9 @@ VERSIONS_HISTORY_URL = 'http://cros-omahaproxy.appspot.com/history' def DisplayBetas(betas): - print 'List of betas from', VERSIONS_HISTORY_URL + print('List of betas from %s' % VERSIONS_HISTORY_URL) for beta in betas: - print ' Release', beta['chrome_major_version'], beta + print(' Release', beta['chrome_major_version'], beta) return @@ -53,26 +55,27 @@ def FindAllBetas(all_versions): def SerializeBetas(all_betas, serialize_file): with open(serialize_file, 'wb') as f: pickle.dump(all_betas, f) - print 'Serialized list of betas into', serialize_file + print('Serialized list of betas into', serialize_file) return def Main(argv): """Get ChromeOS first betas list from history URL.""" - parser = optparse.OptionParser() - parser.add_option('--serialize', - dest='serialize', - default=None, - help='Save list of common images into the specified file.') - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + parser.add_argument('--serialize', + dest='serialize', + default=None, + help='Save list of common images into the specified ' + 'file.') + options = parser.parse_args(argv) try: opener = urllib.URLopener() all_versions = opener.open(VERSIONS_HISTORY_URL) except IOError as ioe: - print 'Cannot open', VERSIONS_HISTORY_URL - print ioe + print('Cannot open', VERSIONS_HISTORY_URL) + print(ioe) return 1 all_betas = FindAllBetas(all_versions) @@ -85,5 +88,5 @@ def Main(argv): if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/heat_map.py b/heat_map.py index 7d4be65f..ae234b51 100755 --- a/heat_map.py +++ b/heat_map.py @@ -1,9 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Copyright 2015 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Wrapper to generate heat maps for chrome.""" +from __future__ import print_function + import argparse import shutil import os @@ -11,9 +13,7 @@ import sys import tempfile from sets import Set -from utils import command_executer -from utils import misc - +from cros_utils import command_executer def IsARepoRoot(directory): """Returns True if directory is the root of a repo checkout.""" @@ -31,6 +31,10 @@ class HeatMapProducer(object): self.binary = binary self.tempDir = '' self.ce = command_executer.GetCommandExecuter() + self.loading_address = None + self.temp_perf = '' + self.temp_perf_inchroot = '' + self.perf_report = '' def copyFileToChroot(self): self.tempDir = tempfile.mkdtemp( @@ -41,7 +45,8 @@ class HeatMapProducer(object): os.path.basename(self.tempDir)) def getPerfReport(self): - cmd = 'cd %s; perf report -D -i perf.data > perf_report.txt' % self.temp_perf_inchroot + cmd = ('cd %s; perf report -D -i perf.data > perf_report.txt' % + self.temp_perf_inchroot) retval = self.ce.ChrootRunCommand(self.chromeos_root, cmd) if retval: raise RuntimeError('Failed to generate perf report') @@ -89,7 +94,7 @@ def main(argv): """Parse the options. Args: - argv: The options with which this script was invoked. + argv: The options with which this script was invoked. Returns: 0 unless an exception is raised. @@ -132,7 +137,7 @@ def main(argv): print('\nheat map and time histgram genereated in the current directory ' 'with name heat_map.png and timeline.png accordingly.') except RuntimeError, e: - print e + print(e) finally: heatmap_producer.RemoveFiles() diff --git a/image_chromeos.py b/image_chromeos.py index 562731a6..1c332d53 100755 --- a/image_chromeos.py +++ b/image_chromeos.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2011 Google Inc. All Rights Reserved. """Script to image a ChromeOS device. @@ -6,11 +6,13 @@ This script images a remote ChromeOS device with a specific image." """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' +import argparse import filecmp import glob -import optparse import os import re import shutil @@ -18,18 +20,18 @@ import sys import tempfile import time -from utils import command_executer -from utils import locks -from utils import logger -from utils import misc -from utils.file_utils import FileUtils +from cros_utils import command_executer +from cros_utils import locks +from cros_utils import logger +from cros_utils import misc +from cros_utils.file_utils import FileUtils checksum_file = '/usr/local/osimage_checksum_file' lock_file = '/tmp/image_chromeos_lock/image_chromeos_lock' def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) @@ -39,54 +41,54 @@ def CheckForCrosFlash(chromeos_root, remote, log_level): # Check to see if remote machine has cherrypy, ctypes command = "python -c 'import cherrypy, ctypes'" - retval = cmd_executer.CrosRunCommand(command, - chromeos_root=chromeos_root, - machine=remote) + ret = cmd_executer.CrosRunCommand(command, + chromeos_root=chromeos_root, + machine=remote) logger.GetLogger().LogFatalIf( - retval == 255, 'Failed ssh to %s (for checking cherrypy)' % remote) + ret == 255, 'Failed ssh to %s (for checking cherrypy)' % remote) logger.GetLogger().LogFatalIf( - retval != 0, "Failed to find cherrypy or ctypes on remote '{}', " + ret != 0, "Failed to find cherrypy or ctypes on remote '{}', " 'cros flash cannot work.'.format(remote)) def DoImage(argv): """Image ChromeOS.""" - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - help='Target directory for ChromeOS installation.') - parser.add_option('-r', '--remote', dest='remote', help='Target device.') - parser.add_option('-i', '--image', dest='image', help='Image binary file.') - parser.add_option('-b', - '--board', - dest='board', - help='Target board override.') - parser.add_option('-f', - '--force', - dest='force', - action='store_true', - default=False, - help='Force an image even if it is non-test.') - parser.add_option('-n', - '--no_lock', - dest='no_lock', - default=False, - action='store_true', - help='Do not attempt to lock remote before imaging. ' - 'This option should only be used in cases where the ' - 'exclusive lock has already been acquired (e.g. in ' - 'a script that calls this one).') - parser.add_option('-l', - '--logging_level', - dest='log_level', - default='verbose', - help='Amount of logging to be used. Valid levels are ' - "'quiet', 'average', and 'verbose'.") - parser.add_option('-a', '--image_args', dest='image_args') - - options = parser.parse_args(argv[1:])[0] + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='Target directory for ChromeOS installation.') + parser.add_argument('-r', '--remote', dest='remote', help='Target device.') + parser.add_argument('-i', '--image', dest='image', help='Image binary file.') + parser.add_argument('-b', + '--board', + dest='board', + help='Target board override.') + parser.add_argument('-f', + '--force', + dest='force', + action='store_true', + default=False, + help='Force an image even if it is non-test.') + parser.add_argument('-n', + '--no_lock', + dest='no_lock', + default=False, + action='store_true', + help='Do not attempt to lock remote before imaging. ' + 'This option should only be used in cases where the ' + 'exclusive lock has already been acquired (e.g. in ' + 'a script that calls this one).') + parser.add_argument('-l', + '--logging_level', + dest='log_level', + default='verbose', + help='Amount of logging to be used. Valid levels are ' + "'quiet', 'average', and 'verbose'.") + parser.add_argument('-a', '--image_args', dest='image_args') + + options = parser.parse_args(argv[1:]) if not options.log_level in command_executer.LOG_LEVEL: Usage(parser, "--logging_level must be 'quiet', 'average' or 'verbose'") @@ -130,7 +132,7 @@ def DoImage(argv): should_unlock = False if not options.no_lock: try: - status = locks.AcquireLock( + _ = locks.AcquireLock( list(options.remote.split()), options.chromeos_root) should_unlock = True except Exception as e: @@ -143,7 +145,7 @@ def DoImage(argv): image_checksum = FileUtils().Md5File(image, log_level=log_level) command = 'cat ' + checksum_file - retval, device_checksum, _ = cmd_executer.CrosRunCommandWOutput( + ret, device_checksum, _ = cmd_executer.CrosRunCommandWOutput( command, chromeos_root=options.chromeos_root, machine=options.remote) @@ -166,8 +168,8 @@ def DoImage(argv): located_image, log_level) if not is_test_image and not options.force: - logger.GetLogger().LogFatal('Have to pass --force to image a non-test' - ' image!') + logger.GetLogger().LogFatal('Have to pass --force to image a ' + 'non-test image!') else: reimage = True found = True @@ -213,19 +215,19 @@ def DoImage(argv): l.LogOutput('CMD : %s' % command) elif log_level == 'average': cmd_executer.SetLogLevel('verbose') - retval = cmd_executer.ChrootRunCommand(options.chromeos_root, - command, - command_timeout=1800) + ret = cmd_executer.ChrootRunCommand(options.chromeos_root, + command, + command_timeout=1800) retries = 0 - while retval != 0 and retries < 2: + while ret != 0 and retries < 2: retries += 1 if log_level == 'quiet': l.LogOutput('Imaging failed. Retry # %d.' % retries) l.LogOutput('CMD : %s' % command) - retval = cmd_executer.ChrootRunCommand(options.chromeos_root, - command, - command_timeout=1800) + ret = cmd_executer.ChrootRunCommand(options.chromeos_root, + command, + command_timeout=1800) if log_level == 'average': cmd_executer.SetLogLevel(log_level) @@ -235,40 +237,42 @@ def DoImage(argv): l.LogOutput('Deleting temp image dir: %s' % temp_dir) shutil.rmtree(temp_dir) - logger.GetLogger().LogFatalIf(retval, 'Image command failed') + logger.GetLogger().LogFatalIf(ret, 'Image command failed') # Unfortunately cros_image_to_target.py sometimes returns early when the # machine isn't fully up yet. - retval = EnsureMachineUp(options.chromeos_root, options.remote, log_level) + ret = EnsureMachineUp(options.chromeos_root, options.remote, log_level) - # If this is a non-local image, then the retval returned from + # If this is a non-local image, then the ret returned from # EnsureMachineUp is the one that will be returned by this function; - # in that case, make sure the value in 'retval' is appropriate. - if not local_image and retval == True: - retval = 0 + # in that case, make sure the value in 'ret' is appropriate. + if not local_image and ret == True: + ret = 0 else: - retval = 1 + ret = 1 if local_image: if log_level == 'average': l.LogOutput('Verifying image.') command = 'echo %s > %s && chmod -w %s' % (image_checksum, - checksum_file, checksum_file) - retval = cmd_executer.CrosRunCommand( + checksum_file, + checksum_file) + ret = cmd_executer.CrosRunCommand( command, chromeos_root=options.chromeos_root, machine=options.remote) - logger.GetLogger().LogFatalIf(retval, 'Writing checksum failed.') + logger.GetLogger().LogFatalIf(ret, 'Writing checksum failed.') - successfully_imaged = VerifyChromeChecksum(options.chromeos_root, image, - options.remote, log_level) + successfully_imaged = VerifyChromeChecksum(options.chromeos_root, + image, options.remote, + log_level) logger.GetLogger().LogFatalIf(not successfully_imaged, 'Image verification failed!') TryRemountPartitionAsRW(options.chromeos_root, options.remote, log_level) else: l.LogOutput('Checksums match. Skipping reimage') - return retval + return ret finally: if should_unlock: locks.ReleaseLock(list(options.remote.split()), options.chromeos_root) @@ -293,7 +297,8 @@ def LocateOrCopyImage(chromeos_root, image, board=None): images_list = glob.glob(images_glob) for potential_image in images_list: if filecmp.cmp(potential_image, image): - l.LogOutput('Found matching image %s in chromeos_root.' % potential_image) + l.LogOutput('Found matching image %s in chromeos_root.' % + potential_image) return [True, potential_image] # We did not find an image. Copy it in the src dir and return the copied # file. @@ -332,9 +337,9 @@ def MountImage(chromeos_root, command = GetImageMountCommand(chromeos_root, image, rootfs_mp, stateful_mp) if unmount: command = '%s --unmount' % command - retval = cmd_executer.RunCommand(command) - logger.GetLogger().LogFatalIf(retval, 'Mount/unmount command failed!') - return retval + ret = cmd_executer.RunCommand(command) + logger.GetLogger().LogFatalIf(ret, 'Mount/unmount command failed!') + return ret def IsImageModdedForTest(chromeos_root, image, log_level): @@ -387,9 +392,10 @@ def TryRemountPartitionAsRW(chromeos_root, remote, log_level): l = logger.GetLogger() cmd_executer = command_executer.GetCommandExecuter(log_level=log_level) command = 'sudo mount -o remount,rw /' - retval = cmd_executer.CrosRunCommand(\ - command, chromeos_root=chromeos_root, machine=remote, terminated_timeout=10) - if retval: + ret = cmd_executer.CrosRunCommand(\ + command, chromeos_root=chromeos_root, machine=remote, + terminated_timeout=10) + if ret: ## Safely ignore. l.LogWarning('Failed to remount partition as rw, ' 'probably the image was not built with ' @@ -412,10 +418,10 @@ def EnsureMachineUp(chromeos_root, remote, log_level): l.LogError('Timeout of %ss reached. Machine still not up. Aborting.' % timeout) return False - retval = cmd_executer.CrosRunCommand(command, - chromeos_root=chromeos_root, - machine=remote) - if not retval: + ret = cmd_executer.CrosRunCommand(command, + chromeos_root=chromeos_root, + machine=remote) + if not ret: return True diff --git a/lock_machine_test.py b/lock_machine_test.py index 7634e2a5..0ffe094d 100644 --- a/lock_machine_test.py +++ b/lock_machine_test.py @@ -4,91 +4,94 @@ MachineManagerTest tests MachineManager. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' from multiprocessing import Process import time import unittest -import lock_machine +import file_lock_machine def LockAndSleep(machine): - lock_machine.Machine(machine, auto=True).Lock(exclusive=True) + file_lock_machine.Machine(machine, auto=True).Lock(exclusive=True) time.sleep(1) class MachineTest(unittest.TestCase): + """Class for testing machine locking.""" def setUp(self): pass def testRepeatedUnlock(self): - mach = lock_machine.Machine('qqqraymes.mtv') - for i in range(10): + mach = file_lock_machine.Machine('qqqraymes.mtv') + for _ in range(10): self.assertFalse(mach.Unlock()) - mach = lock_machine.Machine('qqqraymes.mtv', auto=True) - for i in range(10): + mach = file_lock_machine.Machine('qqqraymes.mtv', auto=True) + for _ in range(10): self.assertFalse(mach.Unlock()) def testLockUnlock(self): - mach = lock_machine.Machine('otter.mtv', '/tmp') - for i in range(10): + mach = file_lock_machine.Machine('otter.mtv', '/tmp') + for _ in range(10): self.assertTrue(mach.Lock(exclusive=True)) self.assertTrue(mach.Unlock(exclusive=True)) - mach = lock_machine.Machine('otter.mtv', '/tmp', True) - for i in range(10): + mach = file_lock_machine.Machine('otter.mtv', '/tmp', True) + for _ in range(10): self.assertTrue(mach.Lock(exclusive=True)) self.assertTrue(mach.Unlock(exclusive=True)) def testSharedLock(self): - mach = lock_machine.Machine('chrotomation.mtv') - for i in range(10): + mach = file_lock_machine.Machine('chrotomation.mtv') + for _ in range(10): self.assertTrue(mach.Lock(exclusive=False)) - for i in range(10): + for _ in range(10): self.assertTrue(mach.Unlock(exclusive=False)) self.assertTrue(mach.Lock(exclusive=True)) self.assertTrue(mach.Unlock(exclusive=True)) - mach = lock_machine.Machine('chrotomation.mtv', auto=True) - for i in range(10): + mach = file_lock_machine.Machine('chrotomation.mtv', auto=True) + for _ in range(10): self.assertTrue(mach.Lock(exclusive=False)) - for i in range(10): + for _ in range(10): self.assertTrue(mach.Unlock(exclusive=False)) self.assertTrue(mach.Lock(exclusive=True)) self.assertTrue(mach.Unlock(exclusive=True)) def testExclusiveLock(self): - mach = lock_machine.Machine('atree.mtv') + mach = file_lock_machine.Machine('atree.mtv') self.assertTrue(mach.Lock(exclusive=True)) - for i in range(10): + for _ in range(10): self.assertFalse(mach.Lock(exclusive=True)) self.assertFalse(mach.Lock(exclusive=False)) self.assertTrue(mach.Unlock(exclusive=True)) - mach = lock_machine.Machine('atree.mtv', auto=True) + mach = file_lock_machine.Machine('atree.mtv', auto=True) self.assertTrue(mach.Lock(exclusive=True)) - for i in range(10): + for _ in range(10): self.assertFalse(mach.Lock(exclusive=True)) self.assertFalse(mach.Lock(exclusive=False)) self.assertTrue(mach.Unlock(exclusive=True)) def testExclusiveState(self): - mach = lock_machine.Machine('testExclusiveState') + mach = file_lock_machine.Machine('testExclusiveState') self.assertTrue(mach.Lock(exclusive=True)) - for i in range(10): + for _ in range(10): self.assertFalse(mach.Lock(exclusive=False)) self.assertTrue(mach.Unlock(exclusive=True)) - mach = lock_machine.Machine('testExclusiveState', auto=True) + mach = file_lock_machine.Machine('testExclusiveState', auto=True) self.assertTrue(mach.Lock(exclusive=True)) - for i in range(10): + for _ in range(10): self.assertFalse(mach.Lock(exclusive=False)) self.assertTrue(mach.Unlock(exclusive=True)) def testAutoLockGone(self): - mach = lock_machine.Machine('lockgone', auto=True) + mach = file_lock_machine.Machine('lockgone', auto=True) p = Process(target=LockAndSleep, args=('lockgone',)) p.start() time.sleep(1.1) @@ -96,7 +99,7 @@ class MachineTest(unittest.TestCase): self.assertTrue(mach.Lock(exclusive=True)) def testAutoLockFromOther(self): - mach = lock_machine.Machine('other_lock', auto=True) + mach = file_lock_machine.Machine('other_lock', auto=True) p = Process(target=LockAndSleep, args=('other_lock',)) p.start() time.sleep(0.5) @@ -106,7 +109,7 @@ class MachineTest(unittest.TestCase): self.assertTrue(mach.Lock(exclusive=True)) def testUnlockByOthers(self): - mach = lock_machine.Machine('other_unlock', auto=True) + mach = file_lock_machine.Machine('other_unlock', auto=True) p = Process(target=LockAndSleep, args=('other_unlock',)) p.start() time.sleep(0.5) diff --git a/produce_output.py b/produce_output.py index 99e48e96..30deea8c 100755 --- a/produce_output.py +++ b/produce_output.py @@ -1,33 +1,27 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """This simulates a real job by producing a lot of output. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' -import optparse -import os -import re -import sys import time -from utils import command_executer -def Main(argv): +def Main(): """The main function.""" - parser = optparse.OptionParser() - - (options, args) = parser.parse_args(argv) for j in range(10): for i in range(10000): - print str(j) + 'The quick brown fox jumped over the lazy dog.' + str(i) + print(str(j) + 'The quick brown fox jumped over the lazy dog.' + str(i)) time.sleep(60) return 0 if __name__ == '__main__': - Main(sys.argv) + Main() diff --git a/remote_gcc_build.py b/remote_gcc_build.py index 807ace37..2e010983 100755 --- a/remote_gcc_build.py +++ b/remote_gcc_build.py @@ -1,10 +1,12 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Script to use remote try-bot build image with local gcc.""" +from __future__ import print_function + import argparse import glob import os @@ -15,15 +17,16 @@ import sys import tempfile import time -from utils import command_executer -from utils import logger -from utils import manifest_versions -from utils import misc +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import manifest_versions +from cros_utils import misc BRANCH = 'the_actual_branch_used_in_this_script' TMP_BRANCH = 'tmp_branch' SLEEP_TIME = 600 +# pylint: disable=anomalous-backslash-in-string def GetPatchNum(output): lines = output.splitlines() @@ -131,10 +134,11 @@ def DownloadImage(target, index, dest, version): os.makedirs(dest) rversion = manifest_versions.RFormatCrosVersion(version) + print(str(rversion)) # ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}" # .format(target, rversion, index)) ls_cmd = ('gsutil ls gs://chromeos-image-archive/trybot-{0}/*-b{2}' - .format(target, rversion, index)) + .format(target, index)) download_cmd = ('$(which gsutil) cp {0} {1}'.format('{0}', dest)) ce = command_executer.GetCommandExecuter() diff --git a/remote_kill_test.py b/remote_kill_test.py index 88aaf15d..71a66907 100755 --- a/remote_kill_test.py +++ b/remote_kill_test.py @@ -1,44 +1,45 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to wrap test_that script. Run this script and kill it. Then run ps -ef to see if sleep -is still running,. +is still running,. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' -import optparse +import argparse import os -import re import sys -import subprocess -from utils import command_executer +from cros_utils import command_executer def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) def Main(argv): - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - help='ChromeOS root checkout directory') - parser.add_option('-r', - '--remote', - dest='remote', - help='Remote chromeos device.') - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='ChromeOS root checkout directory') + parser.add_argument('-r', + '--remote', + dest='remote', + help='Remote chromeos device.') + + _ = parser.parse_args(argv) ce = command_executer.GetCommandExecuter() ce.RunCommand('ls; sleep 10000', machine=os.uname()[1]) return 0 if __name__ == '__main__': - Main(sys.argv) + Main(sys.argv[1:]) diff --git a/remote_test.py b/remote_test.py index 59eb9eb5..82f54ed5 100755 --- a/remote_test.py +++ b/remote_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to wrap test_that script. @@ -6,34 +6,35 @@ This script can login to the chromeos machine using the test private key. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' -import optparse +import argparse import os -import re import sys -from utils import command_executer -from utils import misc +from cros_utils import command_executer +from cros_utils import misc def Usage(parser, message): - print 'ERROR: ' + message + print('ERROR: %s' % message) parser.print_help() sys.exit(0) def Main(argv): - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - help='ChromeOS root checkout directory') - parser.add_option('-r', - '--remote', - dest='remote', - help='Remote chromeos device.') - options = parser.parse_args(argv)[0] + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + help='ChromeOS root checkout directory') + parser.add_argument('-r', + '--remote', + dest='remote', + help='Remote chromeos device.') + options = parser.parse_args(argv) if options.chromeos_root is None: Usage(parser, 'chromeos_root must be given') @@ -89,9 +90,9 @@ def Main(argv): src_cros=True, chromeos_root=options.chromeos_root) board = ce.CrosLearnBoard(options.chromeos_root, options.remote) - print board + print(board) return 0 if __name__ == '__main__': - Main(sys.argv) + Main(sys.argv[1:]) diff --git a/repo_to_repo_files/binutils-master.rtr b/repo_to_repo_files/binutils-master.rtr deleted file mode 100644 index a10c090f..00000000 --- a/repo_to_repo_files/binutils-master.rtr +++ /dev/null @@ -1,15 +0,0 @@ -{ - "input": [{ - "type": "p4", - "address": "perforce2:2666", - "mappings": [ - "//depot2/branches/mobile_toolchain_v15_release_branch/gcctools/google_vendor_src_branch/binutils/binutils-2.21/... ...", - ], - },], - "output": [{ - "type": "git", - "address": "https://chromium.googlesource.com/chromiumos/third_party/binutils.git", - "ignores": [".svn", ".git"], - "gerrit": True, - },], -} diff --git a/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr b/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr deleted file mode 100644 index a8105e0d..00000000 --- a/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr +++ /dev/null @@ -1,16 +0,0 @@ -{ - "input": [{ - "type": "p4", - "address": "perforce2:2666", - "mappings": [ - "//depot2/branches/mobile_toolchain_v16_release_branch/gcctools/google_vendor_src_branch/binutils/binutils-2.22/... ...", - ], - },], - "output": [{ - "type": "git", - "address": "https://chromium.googlesource.com/chromiumos/third_party/binutils.git", - "ignores": [".svn", ".git"], - "branch": "mobile_toolchain_v16_release_branch", - "gerrit": True, - },], -} diff --git a/repo_to_repo_files/crosperf.rtr b/repo_to_repo_files/crosperf.rtr deleted file mode 100644 index 6e6413e9..00000000 --- a/repo_to_repo_files/crosperf.rtr +++ /dev/null @@ -1,31 +0,0 @@ -{ - "input": [{ - "type": "p4", - "address": "perforce2:2666", - "mappings": [ - "//depot2/gcctools/chromeos/v14/crosperf/... crosperf/...", - "//depot2/gcctools/chromeos/v14/utils/__init__.py utils/__init__.py", - "//depot2/gcctools/chromeos/v14/utils/colortrans.py utils/colortrans.py", - "//depot2/gcctools/chromeos/v14/utils/command_executer.py utils/command_executer.py", - "//depot2/gcctools/chromeos/v14/utils/email_sender.py utils/email_sender.py", - "//depot2/gcctools/chromeos/v14/utils/file_utils.py utils/file_utils.py", - "//depot2/gcctools/chromeos/v14/utils/logger.py utils/logger.py", - "//depot2/gcctools/chromeos/v14/utils/misc.py utils/misc.py", - "//depot2/gcctools/chromeos/v14/utils/misc_test.py utils/misc_test.py", - "//depot2/gcctools/chromeos/v14/utils/stats.py utils/stats.py", - "//depot2/gcctools/chromeos/v14/utils/tabulator.py utils/tabulator.py", - "//depot2/gcctools/chromeos/v14/utils/tabulator_test.py utils/tabulator_test.py", - "//depot2/gcctools/chromeos/v14/utils/timeline.py utils/timeline.py", - "//depot2/gcctools/chromeos/v14/utils/timeline_test.py utils/timeline_test.py", - "//depot2/gcctools/chromeos/v14/image_chromeos.py image_chromeos.py", - "//depot2/gcctools/chromeos/v14/lock_machine.py lock_machine.py", - "//depot2/gcctools/chromeos/v14/README README", - ] - },], - "output": [{ - "type": "git", - "address": "https://chromium.googlesource.com/chromiumos/third_party/toolchain-utils.git", - "ignores": [".svn", ".git"], - "gerrit": True, - },], -} diff --git a/repo_to_repo_files/gcc-branches_google_4_7.rtr b/repo_to_repo_files/gcc-branches_google_4_7.rtr deleted file mode 100644 index bbbc87e6..00000000 --- a/repo_to_repo_files/gcc-branches_google_4_7.rtr +++ /dev/null @@ -1,13 +0,0 @@ -{ - "input": [{ - "type": "svn", - "address": "svn://gcc.gnu.org/svn/gcc", - "mappings": ["branches/google/gcc-4_7-mobile"], - },], - "output": [{ - "type": "git", - "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git", - "branch": "gcc.gnu.org/branches/google/gcc-4_7-mobile", - "ignores": [".svn", ".git"], - },], -} diff --git a/repo_to_repo_files/gcc-branches_google_main.rtr b/repo_to_repo_files/gcc-branches_google_main.rtr deleted file mode 100644 index ff8ed0df..00000000 --- a/repo_to_repo_files/gcc-branches_google_main.rtr +++ /dev/null @@ -1,13 +0,0 @@ -{ - "input": [{ - "type": "svn", - "address": "svn://gcc.gnu.org/svn/gcc", - "mappings": ["branches/google/main"], - },], - "output": [{ - "type": "git", - "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git", - "ignores": [".svn", ".git"], - "branch": "gcc.gnu.org/branches/google/main", - },], -} diff --git a/repo_to_repo_files/gcc-master.rtr b/repo_to_repo_files/gcc-master.rtr deleted file mode 100644 index 213e0117..00000000 --- a/repo_to_repo_files/gcc-master.rtr +++ /dev/null @@ -1,14 +0,0 @@ -{ - "input": [{ - "type": "svn", - "address": "svn://gcc.gnu.org/svn/gcc", - "mappings": ["branches/google/gcc-4_7-mobile"], - },], - "output": [{ - "type": "git", - "address": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git", - "branch": "master", - "ignores": [".svn", ".git"], - "gerrit": True, - },], -} diff --git a/repo_to_repo_files/gdb-master.rtr b/repo_to_repo_files/gdb-master.rtr deleted file mode 100644 index aba8fd4a..00000000 --- a/repo_to_repo_files/gdb-master.rtr +++ /dev/null @@ -1,13 +0,0 @@ -{ - "input": [{ - "type": "git", - "branch": "202befe3cb3ad0eec07eaaf3f0706e42926b6bbb", - "address": "git://sourceware.org/git/gdb.git" - },], - "output": [{ - "type": "git", - "branch": "chromeos_master", - "address": "https://chromium.googlesource.com/chromiumos/third_party/gdb.git", - "ignores": [".svn", ".git" ], - },], -} diff --git a/repo_to_repo_files/toolchain-utils.rtr b/repo_to_repo_files/toolchain-utils.rtr deleted file mode 100644 index d228a5c5..00000000 --- a/repo_to_repo_files/toolchain-utils.rtr +++ /dev/null @@ -1,19 +0,0 @@ -{ - "input": [{ - "type": "p4", - "address": "perforce2:2666", - "mappings": [ - "//depot2/gcctools/chromeos/v14/crosperf/... crosperf/...", - "//depot2/gcctools/chromeos/v14/utils/... utils/...", - "//depot2/gcctools/chromeos/v14/image_chromeos.py image_chromeos.py", - "//depot2/gcctools/chromeos/v14/lock_machine.py lock_machine.py", - "//depot2/gcctools/chromeos/v14/README README", - "//depot2/gcctools/chromeos/v14/.gitignore .gitignore", - ] - },], - "output": [{ - "type": "git", - "address": "https://chrome-internal.googlesource.com/chromeos/toolchain-utils.git", - "ignores": [".svn", ".git", ".pyc", "logs"], - },], -} diff --git a/setup_chromeos.py b/setup_chromeos.py index b90e2ae4..b6f9f4df 100755 --- a/setup_chromeos.py +++ b/setup_chromeos.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -9,20 +9,21 @@ This script sets up the ChromeOS source in the given directory, matching a particular release of ChromeOS. """ +from __future__ import print_function + __author__ = 'raymes@google.com (Raymes Khoury)' from datetime import datetime -import getpass -import optparse +import argparse import os import pickle import sys import tempfile import time -from utils import command_executer -from utils import logger -from utils import manifest_versions +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import manifest_versions GCLIENT_FILE = """solutions = [ { "name" : "CHROME_DEPS", @@ -97,14 +98,14 @@ def TimeToCommonVersion(timestamp): def Main(argv): """Checkout the ChromeOS source.""" - parser = optparse.OptionParser() - parser.add_option('--dir', - dest='directory', - help='Target directory for ChromeOS installation.') - parser.add_option('--version', - dest='version', - default='latest_lkgm', - help="""ChromeOS version. Can be: + parser = argparse.ArgumentParser() + parser.add_argument('--dir', + dest='directory', + help='Target directory for ChromeOS installation.') + parser.add_argument('--version', + dest='version', + default='latest_lkgm', + help="""ChromeOS version. Can be: (1) A release version in the format: 'X.X.X.X' (2) 'top' for top of trunk (3) 'latest_lkgm' for the latest lkgm version @@ -112,31 +113,33 @@ def Main(argv): (5) 'latest_common' for the latest team common stable version (6) 'common' for the team common stable version before timestamp Default is 'latest_lkgm'.""") - parser.add_option('--timestamp', - dest='timestamp', - default=None, - help="""Timestamps in epoch format. It will check out the -latest LKGM or the latest COMMON version of ChromeOS before the timestamp. -Use in combination with --version=latest or --version=common. Use -'date -d <date string> +%s' to find epoch time""") - parser.add_option('--minilayout', - dest='minilayout', - default=False, - action='store_true', - help="""Whether to checkout the minilayout -(smaller checkout).'""") - parser.add_option('--jobs', - '-j', - dest='jobs', - help='Number of repo sync threads to use.') - parser.add_option('--public', - '-p', - dest='public', - default=False, - action='store_true', - help='Use the public checkout instead of the private one.') - - options = parser.parse_args(argv)[0] + parser.add_argument('--timestamp', + dest='timestamp', + default=None, + help='Timestamps in epoch format. It will check out the' + 'latest LKGM or the latest COMMON version of ChromeOS' + ' before the timestamp. Use in combination with' + ' --version=latest or --version=common. Use ' + '"date -d <date string> +%s" to find epoch time') + parser.add_argument('--minilayout', + dest='minilayout', + default=False, + action='store_true', + help='Whether to checkout the minilayout (smaller ' + 'checkout).') + parser.add_argument('--jobs', + '-j', + dest='jobs', + help='Number of repo sync threads to use.') + parser.add_argument('--public', + '-p', + dest='public', + default=False, + action='store_true', + help='Use the public checkout instead of the private ' + 'one.') + + options = parser.parse_args(argv) if not options.version: parser.print_help() @@ -165,10 +168,12 @@ Use in combination with --version=latest or --version=common. Use 'chromiumos/manifest-versions.git') else: manifest_repo = ( - 'https://chrome-internal.googlesource.com/chromeos/manifest-internal.git' + 'https://chrome-internal.googlesource.com/chromeos/' + 'manifest-internal.git' ) versions_repo = ( - 'https://chrome-internal.googlesource.com/chromeos/manifest-versions.git' + 'https://chrome-internal.googlesource.com/chromeos/' + 'manifest-versions.git' ) if version == 'top': @@ -243,5 +248,5 @@ Use in combination with --version=latest or --version=common. Use if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval) diff --git a/tc_enter_chroot.py b/tc_enter_chroot.py index 39bb7dc4..573e5585 100755 --- a/tc_enter_chroot.py +++ b/tc_enter_chroot.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script to enter the ChromeOS chroot with mounted sources. @@ -6,21 +6,23 @@ This script enters the chroot with mounted sources. """ +from __future__ import print_function + __author__ = 'asharif@google.com (Ahmad Sharif)' +import argparse import getpass -import optparse import os import pwd -import stat import sys -from utils import command_executer -from utils import logger -from utils import misc +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import misc -class MountPoint: +class MountPoint(object): + """Mount point class""" def __init__(self, external_dir, mount_dir, owner, options=None): self.external_dir = os.path.realpath(external_dir) @@ -29,32 +31,32 @@ class MountPoint: self.options = options def CreateAndOwnDir(self, dir_name): - retval = 0 + retv = 0 if not os.path.exists(dir_name): command = 'mkdir -p ' + dir_name command += ' || sudo mkdir -p ' + dir_name - retval = command_executer.GetCommandExecuter().RunCommand(command) - if retval != 0: - return retval + retv = command_executer.GetCommandExecuter().RunCommand(command) + if retv != 0: + return retv pw = pwd.getpwnam(self.owner) if os.stat(dir_name).st_uid != pw.pw_uid: command = 'sudo chown -f ' + self.owner + ' ' + dir_name - retval = command_executer.GetCommandExecuter().RunCommand(command) - return retval + retv = command_executer.GetCommandExecuter().RunCommand(command) + return retv def DoMount(self): ce = command_executer.GetCommandExecuter() mount_signature = '%s on %s' % (self.external_dir, self.mount_dir) command = 'mount' - retval, out, err = ce.RunCommandWOutput(command) + retv, out, _ = ce.RunCommandWOutput(command) if mount_signature not in out: - retval = self.CreateAndOwnDir(self.mount_dir) - logger.GetLogger().LogFatalIf(retval, 'Cannot create mount_dir!') - retval = self.CreateAndOwnDir(self.external_dir) - logger.GetLogger().LogFatalIf(retval, 'Cannot create external_dir!') - retval = self.MountDir() - logger.GetLogger().LogFatalIf(retval, 'Cannot mount!') - return retval + retv = self.CreateAndOwnDir(self.mount_dir) + logger.GetLogger().LogFatalIf(retv, 'Cannot create mount_dir!') + retv = self.CreateAndOwnDir(self.external_dir) + logger.GetLogger().LogFatalIf(retv, 'Cannot create external_dir!') + retv = self.MountDir() + logger.GetLogger().LogFatalIf(retv, 'Cannot mount!') + return retv else: return 0 @@ -66,8 +68,8 @@ class MountPoint: command = 'sudo mount --bind ' + self.external_dir + ' ' + self.mount_dir if self.options == 'ro': command += ' && sudo mount --bind -oremount,ro ' + self.mount_dir - retval = command_executer.GetCommandExecuter().RunCommand(command) - return retval + retv = command_executer.GetCommandExecuter().RunCommand(command) + return retv def __str__(self): ret = '' @@ -82,43 +84,45 @@ class MountPoint: def Main(argv, return_output=False): """The main function.""" - parser = optparse.OptionParser() - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - default='../..', - help='ChromeOS root checkout directory.') - parser.add_option('-t', - '--toolchain_root', - dest='toolchain_root', - help='Toolchain root directory.') - parser.add_option('-o', - '--output', - dest='output', - help='Toolchain output directory') - parser.add_option('--sudo', - dest='sudo', - action='store_true', - default=False, - help='Run the command with sudo.') - parser.add_option('-r', - '--third_party', - dest='third_party', - help='The third_party directory to mount.') - parser.add_option( - '-m', - '--other_mounts', - dest='other_mounts', - help='Other mount points in the form: ' + 'dir:mounted_dir:options') - parser.add_option('-s', - '--mount-scripts-only', - dest='mount_scripts_only', - action='store_true', - default=False, - help='Mount only the scripts dir, and not the sources.') - - passthrough_argv = [] - (options, passthrough_argv) = parser.parse_args(argv) + + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + default='../..', + help='ChromeOS root checkout directory.') + parser.add_argument('-t', + '--toolchain_root', + dest='toolchain_root', + help='Toolchain root directory.') + parser.add_argument('-o', + '--output', + dest='output', + help='Toolchain output directory') + parser.add_argument('--sudo', + dest='sudo', + action='store_true', + default=False, + help='Run the command with sudo.') + parser.add_argument('-r', + '--third_party', + dest='third_party', + help='The third_party directory to mount.') + parser.add_argument('-m', + '--other_mounts', + dest='other_mounts', + help='Other mount points in the form: ' + 'dir:mounted_dir:options') + parser.add_argument('-s', + '--mount-scripts-only', + dest='mount_scripts_only', + action='store_true', + default=False, + help='Mount only the scripts dir, and not the sources.') + parser.add_argument('passthrough_argv', nargs='*', + help='Command to be executed inside the chroot.') + + options = parser.parse_args(argv) chromeos_root = options.chromeos_root @@ -206,13 +210,13 @@ def Main(argv, return_output=False): mount_points.append(mount_point) for mount_point in mount_points: - retval = mount_point.DoMount() - if retval != 0: - return retval + retv = mount_point.DoMount() + if retv != 0: + return retv # Finally, create the symlink to build-gcc. command = 'sudo chown ' + getpass.getuser() + ' ' + full_mounted_tc_root - retval = command_executer.GetCommandExecuter().RunCommand(command) + retv = command_executer.GetCommandExecuter().RunCommand(command) try: CreateSymlink(last_dir + '/build-gcc', full_mounted_tc_root + '/build-gcc') @@ -224,33 +228,33 @@ def Main(argv, return_output=False): # Now call cros_sdk --enter with the rest of the arguments. command = 'cd %s/src/scripts && cros_sdk --enter' % chromeos_root - if len(passthrough_argv) > 1: - inner_command = ' '.join(passthrough_argv[1:]) + if len(options.passthrough_argv) > 1: + inner_command = ' '.join(options.passthrough_argv[1:]) inner_command = inner_command.strip() if inner_command.startswith('-- '): inner_command = inner_command[3:] command_file = 'tc_enter_chroot.cmd' command_file_path = chromeos_root + '/src/scripts/' + command_file - retval = command_executer.GetCommandExecuter().RunCommand('sudo rm -f ' + - command_file_path) - if retval != 0: - return retval + retv = command_executer.GetCommandExecuter().RunCommand('sudo rm -f ' + + command_file_path) + if retv != 0: + return retv f = open(command_file_path, 'w') f.write(inner_command) f.close() logger.GetLogger().LogCmd(inner_command) - retval = command_executer.GetCommandExecuter().RunCommand('chmod +x ' + - command_file_path) - if retval != 0: - return retval + retv = command_executer.GetCommandExecuter().RunCommand('chmod +x ' + + command_file_path) + if retv != 0: + return retv if options.sudo: command += ' sudo ./' + command_file else: command += ' ./' + command_file - retval = command_executer.GetCommandExecuter().RunCommandGeneric( + retv = command_executer.GetCommandExecuter().RunCommandGeneric( command, return_output) - return retval + return retv else: os.chdir('%s/src/scripts' % chromeos_root) ce = command_executer.GetCommandExecuter() diff --git a/test_gcc_dejagnu.py b/test_gcc_dejagnu.py index 97dbfa2a..37640710 100755 --- a/test_gcc_dejagnu.py +++ b/test_gcc_dejagnu.py @@ -1,26 +1,28 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2010 Google Inc. All Rights Reserved. """Script adapter used by automation client for testing dejagnu. + This is not intended to be run on command line. - To kick off a single dejagnu run, use chromeos/v14/dejagnu/run_dejagnu.py + To kick off a single dejagnu run, use ./dejagnu/run_dejagnu.py """ +from __future__ import print_function + __author__ = 'shenhan@google.com (Han Shen)' -import optparse -import os -from os import path +import argparse import sys import setup_chromeos import build_tc from dejagnu import run_dejagnu -from utils import command_executer -from utils import email_sender +from cros_utils import command_executer +from cros_utils import email_sender class DejagnuAdapter(object): + """Dejagnu Adapter class""" # TODO(shenhan): move these to constants.py. _CHROMIUM_GCC_GIT = ('https://chromium.googlesource.com/' @@ -91,17 +93,17 @@ class DejagnuAdapter(object): # Parse the output log to determine how many failures we have. # Return -1 if parse output log failed. -def GetNumNewFailures(str): - if not str: +def GetNumNewFailures(input_str): + if not input_str: return 0 start_counting = False n_failures = 0 - for l in str.splitlines(): - print l + for l in input_str.splitlines(): + print(l) if not start_counting and 'Build results not in the manifest' in l: start_counting = True elif start_counting and l and ( - l.find('UNRESOLVED:') == 0 or l.find('FAIL:') == 0 or \ + l.find('UNRESOLVED:') == 0 or l.find('FAIL:') == 0 or l.find('XFAIL:') == 0 or l.find('XPASS:') == 0): n_failures = n_failures + 1 if not start_counting: @@ -112,7 +114,6 @@ def GetNumNewFailures(str): # Do not throw any exception in this function! def EmailResult(result): email_to = ['c-compiler-chrome@google.com'] - email_from = ['dejagnu-job@google.com'] if len(result) == 4: subject = 'Job failed: dejagnu test didn\'t finish' email_text = 'Job failed prematurely, check exception below.\n' + \ @@ -139,7 +140,7 @@ def EmailResult(result): try: email_sender.EmailSender().SendEmail(email_to, subject, email_text) - print 'Email sent.' + print('Email sent.') except Exception as e: # Do not propagate this email sending exception, you want to email an # email exception? Just log it on console. @@ -151,41 +152,41 @@ def EmailResult(result): def ProcessArguments(argv): """Processing script arguments.""" - parser = optparse.OptionParser( + parser = argparse.ArgumentParser( description=('This script is used by nightly client to test gcc. ' 'DO NOT run it unless you know what you are doing.'), usage='test_gcc_dejagnu.py options') - parser.add_option('-b', - '--board', - dest='board', - help=('Required. Specify board type. For example ' - '\'lumpy\' and \'daisy\'')) - parser.add_option('-r', - '--remote', - dest='remote', - help=('Required. Specify remote board address')) - parser.add_option('-g', - '--gcc_dir', - dest='gcc_dir', - default='gcc.live', - help=('Optional. Specify gcc checkout directory.')) - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - default='chromeos.live', - help=('Optional. Specify chromeos checkout directory.')) - parser.add_option('--cleanup', - dest='cleanup', - default=None, - help=('Optional. Do cleanup after the test.')) - parser.add_option('--runtestflags', - dest='runtestflags', - default=None, - help=('Optional. Options to RUNTESTFLAGS env var ' - 'while invoking make check. ' - '(Mainly used for testing purpose.)')) - - options, args = parser.parse_args(argv) + parser.add_argument('-b', + '--board', + dest='board', + help=('Required. Specify board type. For example ' + '\'lumpy\' and \'daisy\'')) + parser.add_argument('-r', + '--remote', + dest='remote', + help=('Required. Specify remote board address')) + parser.add_argument('-g', + '--gcc_dir', + dest='gcc_dir', + default='gcc.live', + help=('Optional. Specify gcc checkout directory.')) + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + default='chromeos.live', + help=('Optional. Specify chromeos checkout directory.')) + parser.add_argument('--cleanup', + dest='cleanup', + default=None, + help=('Optional. Do cleanup after the test.')) + parser.add_argument('--runtestflags', + dest='runtestflags', + default=None, + help=('Optional. Options to RUNTESTFLAGS env var ' + 'while invoking make check. ' + '(Mainly used for testing purpose.)')) + + options = parser.parse_args(argv[1:]) if not options.board or not options.remote: raise Exception('--board and --remote are mandatory options.') @@ -204,11 +205,12 @@ def Main(argv): adapter.BuildGCC() ret = adapter.CheckGCC() except Exception as e: - print e + print(e) ret = (1, '', '', str(e)) finally: EmailResult(ret) - return ret + + return ret if __name__ == '__main__': diff --git a/test_gdb_dejagnu.py b/test_gdb_dejagnu.py index 65f007ef..a1c44dc9 100755 --- a/test_gdb_dejagnu.py +++ b/test_gdb_dejagnu.py @@ -1,15 +1,23 @@ -#!/usr/bin/python +#!/usr/bin/python2 +"""Script adapter used by automation client for testing dejagnu. -import optparse + This is not intended to be run on command line. + To kick off a single dejagnu run, use ./dejagnu/run_dejagnu.py +""" + +from __future__ import print_function + +import argparse import sys import setup_chromeos from dejagnu import gdb_dejagnu -from utils import command_executer -from utils import email_sender +from cros_utils import command_executer +from cros_utils import email_sender class DejagnuAdapter(object): + """Dejagnu Adapter class.""" def __init__(self, board, remote, gdb_dir, chromeos_root, cleanup): self._board = board @@ -84,7 +92,7 @@ def EmailResult(result): try: email_sender.EmailSender().SendEmail(email_to, subject, email_text) - print 'Email sent.' + print('Email sent.') except Exception as e: # Do not propagate this email sending exception, you want to email an # email exception? Just log it on console. @@ -96,35 +104,35 @@ def EmailResult(result): def ProcessArguments(argv): """Processing script arguments.""" - parser = optparse.OptionParser( + parser = argparse.ArgumentParser( description=('This script is used by nightly client to test gdb. ' 'DO NOT run it unless you know what you are doing.'), usage='test_gdb_dejagnu.py options') - parser.add_option('-b', - '--board', - dest='board', - help=('Required. Specify board type. For example ' - '\'lumpy\' and \'daisy\'')) - parser.add_option('-r', - '--remote', - dest='remote', - help=('Required. Specify remote board address')) - parser.add_option('-g', - '--gdb_dir', - dest='gdb_dir', - default='', - help=('Optional. Specify gdb checkout directory.')) - parser.add_option('-c', - '--chromeos_root', - dest='chromeos_root', - default='chromeos.live', - help=('Optional. Specify chromeos checkout directory.')) - parser.add_option('--cleanup', - dest='cleanup', - default=None, - help=('Optional. Do cleanup after the test.')) - - options, _ = parser.parse_args(argv) + parser.add_argument('-b', + '--board', + dest='board', + help=('Required. Specify board type. For example ' + '\'lumpy\' and \'daisy\'')) + parser.add_argument('-r', + '--remote', + dest='remote', + help=('Required. Specify remote board address')) + parser.add_argument('-g', + '--gdb_dir', + dest='gdb_dir', + default='', + help=('Optional. Specify gdb checkout directory.')) + parser.add_argument('-c', + '--chromeos_root', + dest='chromeos_root', + default='chromeos.live', + help=('Optional. Specify chromeos checkout directory.')) + parser.add_argument('--cleanup', + dest='cleanup', + default=None, + help=('Optional. Do cleanup after the test.')) + + options = parser.parse_args(argv) if not options.board or not options.remote: raise Exception('--board and --remote are mandatory options.') @@ -134,7 +142,7 @@ def ProcessArguments(argv): def Main(argv): opt = ProcessArguments(argv) - print opt + print(opt) adapter = DejagnuAdapter(opt.board, opt.remote, opt.gdb_dir, opt.chromeos_root, opt.cleanup) try: @@ -142,13 +150,14 @@ def Main(argv): adapter.SetupBoard() ret = adapter.CheckGDB() except Exception as e: - print e + print(e) ret = (1, '', '', str(e)) finally: EmailResult(ret) - return ret + + return ret if __name__ == '__main__': - retval = Main(sys.argv) + retval = Main(sys.argv[1:]) sys.exit(retval[0]) diff --git a/test_toolchains.py b/test_toolchains.py index fdd13fe5..65c359a6 100755 --- a/test_toolchains.py +++ b/test_toolchains.py @@ -1,17 +1,20 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Script to test different toolchains against ChromeOS benchmarks. +"""Toolchain team nightly performance test script (local builds).""" + +from __future__ import print_function + +import argparse import datetime -import optparse import os -import string import sys import build_chromeos import setup_chromeos import time -from utils import command_executer -from utils import misc -from utils import logger +from cros_utils import command_executer +from cros_utils import misc +from cros_utils import logger CROSTC_ROOT = '/usr/local/google/crostc' MAIL_PROGRAM = '~/var/bin/mail-sheriff' @@ -21,18 +24,21 @@ NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') class GCCConfig(object): + """GCC configuration class.""" def __init__(self, githash): self.githash = githash -class ToolchainConfig: +class ToolchainConfig(object): + """Toolchain configuration class.""" - def __init__(self, gcc_config=None, binutils_config=None): + def __init__(self, gcc_config=None): self.gcc_config = gcc_config class ChromeOSCheckout(object): + """Main class for checking out, building and testing ChromeOS.""" def __init__(self, board, chromeos_root): self._board = board @@ -51,7 +57,9 @@ class ChromeOSCheckout(object): return self._ce.RunCommand(command) def _GetBuildNumber(self): - """ This function assumes a ChromeOS image has been built in the chroot. + """Get the build number of the ChromeOS image from the chroot. + + This function assumes a ChromeOS image has been built in the chroot. It translates the 'latest' symlink in the <chroot>/src/build/images/<board> directory, to find the actual ChromeOS build number for the image that was built. For example, if @@ -74,7 +82,7 @@ class ChromeOSCheckout(object): image_parts = last_piece.split('.') self._build_num = image_parts[0] - def _BuildLabelName(self, config, board): + def _BuildLabelName(self, config): pieces = config.split('/') compiler_version = pieces[-1] label = compiler_version + '_tot_afdo' @@ -148,6 +156,7 @@ class ChromeOSCheckout(object): class ToolchainComparator(ChromeOSCheckout): + """Main class for running tests and generating reports.""" def __init__(self, board, @@ -182,16 +191,16 @@ class ToolchainComparator(ChromeOSCheckout): 'googlestorage_account.boto') # Copy the file to the correct place copy_cmd = 'cp %s %s' % (src, dest) - retval = self._ce.RunCommand(copy_cmd) - if retval != 0: + retv = self._ce.RunCommand(copy_cmd) + if retv != 0: raise RuntimeError("Couldn't copy .boto file for google storage.") # Fix protections on ssh key command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' '/chrome-src-internal/src/third_party/chromite/ssh_keys' '/testing_rsa') - retval = self._ce.ChrootRunCommand(self._chromeos_root, command) - if retval != 0: + retv = self._ce.ChrootRunCommand(self._chromeos_root, command) + if retv != 0: raise RuntimeError('chmod for testing_rsa failed') def _TestLabels(self, labels): @@ -212,8 +221,8 @@ class ToolchainComparator(ChromeOSCheckout): """ with open(experiment_file, 'w') as f: - print >> f, experiment_header - print >> f, experiment_tests + f.write(experiment_header) + f.write(experiment_tests) for label in labels: # TODO(asharif): Fix crosperf so it accepts labels with symbols crosperf_label = label @@ -232,7 +241,7 @@ class ToolchainComparator(ChromeOSCheckout): build: %s } """ % (self._chromeos_root, build_name) - print >> f, official_image + f.write(official_image) else: experiment_image = """ @@ -243,7 +252,7 @@ class ToolchainComparator(ChromeOSCheckout): """ % (crosperf_label, os.path.join( misc.GetImageDir(self._chromeos_root, self._board), label, 'chromiumos_test_image.bin'), image_args) - print >> f, experiment_image + f.write(experiment_image) crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf') noschedv2_opts = '--noschedv2' if self._noschedv2 else '' @@ -264,7 +273,9 @@ class ToolchainComparator(ChromeOSCheckout): return def _CopyWeeklyReportFiles(self, labels): - """Create tar files of the custom and official images and copy them + """Move files into place for creating 7-day reports. + + Create tar files of the custom and official images and copy them to the weekly reports directory, so they exist when the weekly report gets generated. IMPORTANT NOTE: This function must run *after* crosperf has been run; otherwise the vanilla images will not be there. @@ -310,8 +321,8 @@ class ToolchainComparator(ChromeOSCheckout): labels = [] labels.append('vanilla') for config in self._configs: - label = self._BuildLabelName(config.gcc_config.githash, self._board) - if (not misc.DoesLabelExist(self._chromeos_root, self._board, label)): + label = self._BuildLabelName(config.gcc_config.githash) + if not misc.DoesLabelExist(self._chromeos_root, self._board, label): self._BuildToolchain(config) label = self._BuildAndImage(label) labels.append(label) @@ -335,43 +346,43 @@ def Main(argv): # Common initializations ### command_executer.InitCommandExecuter(True) command_executer.InitCommandExecuter() - parser = optparse.OptionParser() - parser.add_option('--remote', - dest='remote', - help='Remote machines to run tests on.') - parser.add_option('--board', - dest='board', - default='x86-zgb', - help='The target board.') - parser.add_option('--githashes', - dest='githashes', - default='master', - help='The gcc githashes to test.') - parser.add_option('--clean', - dest='clean', - default=False, - action='store_true', - help='Clean the chroot after testing.') - parser.add_option('--public', - dest='public', - default=False, - action='store_true', - help='Use the public checkout/build.') - parser.add_option('--force-mismatch', - dest='force_mismatch', - default='', - help='Force the image regardless of board mismatch') - parser.add_option('--noschedv2', - dest='noschedv2', - action='store_true', - default=False, - help='Pass --noschedv2 to crosperf.') - options, _ = parser.parse_args(argv) + parser = argparse.ArgumentParser() + parser.add_argument('--remote', + dest='remote', + help='Remote machines to run tests on.') + parser.add_argument('--board', + dest='board', + default='x86-alex', + help='The target board.') + parser.add_argument('--githashes', + dest='githashes', + default='master', + help='The gcc githashes to test.') + parser.add_argument('--clean', + dest='clean', + default=False, + action='store_true', + help='Clean the chroot after testing.') + parser.add_argument('--public', + dest='public', + default=False, + action='store_true', + help='Use the public checkout/build.') + parser.add_argument('--force-mismatch', + dest='force_mismatch', + default='', + help='Force the image regardless of board mismatch') + parser.add_argument('--noschedv2', + dest='noschedv2', + action='store_true', + default=False, + help='Pass --noschedv2 to crosperf.') + options = parser.parse_args(argv) if not options.board: - print 'Please give a board.' + print('Please give a board.') return 1 if not options.remote: - print 'Please give at least one remote machine.' + print('Please give at least one remote machine.') return 1 toolchain_configs = [] for githash in options.githashes.split(','): diff --git a/update_telemetry_defaults.py b/update_telemetry_defaults.py index 6d83e143..9ee7d8b0 100755 --- a/update_telemetry_defaults.py +++ b/update_telemetry_defaults.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright 2013 Google Inc. All Rights Reserved. """Script to maintain the Telemetry benchmark default results file. @@ -9,18 +9,21 @@ benchmarks. """ +from __future__ import print_function + __author__ = 'cmtice@google.com (Caroline Tice)' import os import sys import json -from utils import misc +from cros_utils import misc Defaults = {} class TelemetryDefaults(object): + """Class for handling telemetry default return result fields.""" DEFAULTS_FILE_NAME = 'crosperf/default-telemetry-results.json' @@ -111,7 +114,7 @@ class TelemetryDefaults(object): self.ShowOptions() def ShowOptions(self): - print """ + print(""" Below are the valid user options and their arguments, and an explanation of what each option does. You may either print out the full name of the option, or you may use the first letter of the option. Case (upper or @@ -131,7 +134,7 @@ lower) does not matter, for the command (case of the result name DOES matter): (Q)uit - Exit this program, saving changes. (T)erminate - Exit this program; abandon changes. -""" +""") def GetUserInput(self): # Prompt user diff --git a/weekly_report.py b/weekly_report.py index aa53f1d6..e74c623d 100755 --- a/weekly_report.py +++ b/weekly_report.py @@ -1,15 +1,17 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright Google Inc. 2014 +"""Module to generate the 7-day crosperf reports.""" +from __future__ import print_function + +import argparse import datetime -import optparse import os import sys -import time -from utils import constants -from utils import command_executer +from cros_utils import constants +from cros_utils import command_executer WEEKDAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'] DATA_ROOT_DIR = os.path.join(constants.CROSTC_WORKSPACE, 'weekly_test_data') @@ -41,8 +43,8 @@ benchmark: all_toolchain_perf { cmd_executer.RunCommand(cmd) with open(filename, 'w') as f: - print >> f, experiment_header - print >> f, experiment_tests + f.write(experiment_header) + f.write(experiment_tests) # Add each vanilla image for test_path in vanilla_image_paths: @@ -55,7 +57,7 @@ benchmark: all_toolchain_perf { } """ % (test_name, chromeos_root, os.path.join(test_path, 'chromiumos_test_image.bin')) - print >> f, test_image + f.write(test_image) return filename @@ -84,8 +86,8 @@ benchmark: all_toolchain_perf { cmd_executer.RunCommand(cmd) with open(filename, 'w') as f: - print >> f, experiment_header - print >> f, experiment_tests + f.write(experiment_header) + f.write(experiment_tests) # Add vanilla image (first) vanilla_image = """ @@ -96,7 +98,7 @@ benchmark: all_toolchain_perf { """ % (vanilla_image_path.split('/')[-1], chromeos_root, os.path.join(vanilla_image_path, 'chromiumos_test_image.bin')) - print >> f, vanilla_image + f.write(vanilla_image) # Add each test image for test_path in test_image_paths: @@ -109,31 +111,32 @@ benchmark: all_toolchain_perf { } """ % (test_name, chromeos_root, os.path.join(test_path, 'chromiumos_test_image.bin')) - print >> f, test_image + f.write(test_image) return filename def Main(argv): - parser = optparse.OptionParser() - parser.add_option('-b', '--board', dest='board', help='Target board.') - parser.add_option('-r', '--remote', dest='remote', help='Target device.') - parser.add_option('-v', - '--vanilla_only', - dest='vanilla_only', - action='store_true', - default=False, - help='Generate a report comparing only the vanilla images.') + parser = argparse.ArgumentParser() + parser.add_argument('-b', '--board', dest='board', help='Target board.') + parser.add_argument('-r', '--remote', dest='remote', help='Target device.') + parser.add_argument('-v', + '--vanilla_only', + dest='vanilla_only', + action='store_true', + default=False, + help='Generate a report comparing only the vanilla ' + 'images.') - options = parser.parse_args(argv[1:])[0] + options = parser.parse_args(argv[1:]) if not options.board: - print 'Must specify a board.' + print('Must specify a board.') return 1 if not options.remote: - print 'Must specify at least one remote.' + print('Must specify at least one remote.') return 1 cmd_executer = command_executer.GetCommandExecuter(log_level='average') @@ -214,7 +217,7 @@ def Main(argv): break if not chromeos_root: - print 'Unable to locate a usable chroot. Exiting without report.' + print('Unable to locate a usable chroot. Exiting without report.') return 1 # Create the Crosperf experiment file for generating the weekly report. @@ -231,8 +234,8 @@ def Main(argv): cmd = ('%s/toolchain-utils/crosperf/crosperf ' '%s --no_email=True --results_dir=%s' % (constants.CROSTC_WORKSPACE, filename, results_dir)) - retval = cmd_executer.RunCommand(cmd) - if retval == 0: + retv = cmd_executer.RunCommand(cmd) + if retv == 0: # Send the email, if the crosperf command worked. filename = os.path.join(results_dir, 'msg_body.html') if (os.path.exists(filename) and @@ -242,9 +245,9 @@ def Main(argv): vanilla_string = ' Vanilla ' command = ('cat %s | %s -s "Weekly%sReport results, %s" -team -html' % (filename, MAIL_PROGRAM, vanilla_string, options.board)) - retval = cmd_executer.RunCommand(command) + retv = cmd_executer.RunCommand(command) - return retval + return retv if __name__ == '__main__': |