diff options
Diffstat (limited to 'deprecated/fdo_scripts')
| -rwxr-xr-x | deprecated/fdo_scripts/divide_and_merge_profiles.py | 140 | ||||
| -rwxr-xr-x | deprecated/fdo_scripts/divide_and_merge_profiles_test.py | 127 | ||||
| -rwxr-xr-x | deprecated/fdo_scripts/profile_cycler.py | 199 | ||||
| -rw-r--r-- | deprecated/fdo_scripts/summarize_hot_blocks.py | 187 | ||||
| -rw-r--r-- | deprecated/fdo_scripts/vanilla_vs_fdo.py | 312 |
5 files changed, 0 insertions, 965 deletions
diff --git a/deprecated/fdo_scripts/divide_and_merge_profiles.py b/deprecated/fdo_scripts/divide_and_merge_profiles.py deleted file mode 100755 index 5108aa6f..00000000 --- a/deprecated/fdo_scripts/divide_and_merge_profiles.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2011 Google Inc. All Rights Reserved. -"""Script to divide and merge profiles.""" - -import copy -import optparse -import os -import pickle -import re -import sys -import tempfile - -import build_chrome_browser -import lock_machine -import run_tests -from cros_utils import command_executer -from cros_utils import logger - - -class ProfileMerger: - - def __init__(self, inputs, output, chunk_size, merge_program, multipliers): - self._inputs = inputs - self._output = output - self._chunk_size = chunk_size - self._merge_program = merge_program - self._multipliers = multipliers - self._ce = command_executer.GetCommandExecuter() - self._l = logger.GetLogger() - - def _GetFilesSetForInputDir(self, input_dir): - output_file = tempfile.mktemp() - command = "find %s -name '*.gcda' -o -name '*.imports' > %s" % (input_dir, - output_file) - self._ce.RunCommand(command) - files = open(output_file, 'r').read() - files_set = set([]) - for f in files.splitlines(): - stripped_file = f.replace(input_dir, '', 1) - stripped_file = stripped_file.lstrip('/') - files_set.add(stripped_file) - return files_set - - def _PopulateFilesSet(self): - self._files_set = set([]) - for i in self._inputs: - current_files_set = self._GetFilesSetForInputDir(i) - self._files_set.update(current_files_set) - - def _GetSubset(self): - ret = [] - for i in range(self._chunk_size): - if not self._files_set: - break - ret.append(self._files_set.pop()) - return ret - - def _CopyFilesTree(self, input_dir, files, output_dir): - for f in files: - src_file = os.path.join(input_dir, f) - dst_file = os.path.join(output_dir, f) - if not os.path.isdir(os.path.dirname(dst_file)): - command = 'mkdir -p %s' % os.path.dirname(dst_file) - self._ce.RunCommand(command) - command = 'cp %s %s' % (src_file, dst_file) - self._ce.RunCommand(command) - - def _DoChunkMerge(self, current_files): - temp_dirs = [] - for i in self._inputs: - temp_dir = tempfile.mkdtemp() - temp_dirs.append(temp_dir) - self._CopyFilesTree(i, current_files, temp_dir) - # Now do the merge. - command = ('%s --inputs=%s --output=%s' % - (self._merge_program, ','.join(temp_dirs), self._output)) - if self._multipliers: - command = ('%s --multipliers=%s' % (command, self._multipliers)) - ret = self._ce.RunCommand(command) - assert ret == 0, '%s command failed!' % command - for temp_dir in temp_dirs: - command = 'rm -rf %s' % temp_dir - self._ce.RunCommand(command) - - def DoMerge(self): - self._PopulateFilesSet() - while True: - current_files = self._GetSubset() - if not current_files: - break - self._DoChunkMerge(current_files) - - -def Main(argv): - """The main function.""" - # Common initializations - ### command_executer.InitCommandExecuter(True) - command_executer.InitCommandExecuter() - l = logger.GetLogger() - ce = command_executer.GetCommandExecuter() - parser = optparse.OptionParser() - parser.add_option('--inputs', - dest='inputs', - help='Comma-separated input profile directories to merge.') - parser.add_option('--output', dest='output', help='Output profile directory.') - parser.add_option('--chunk_size', - dest='chunk_size', - default='50', - help='Chunk size to divide up the profiles into.') - parser.add_option('--merge_program', - dest='merge_program', - default='/home/xur/bin/profile_merge_v15.par', - help='Merge program to use to do the actual merge.') - parser.add_option('--multipliers', - dest='multipliers', - help='multipliers to use when merging. (optional)') - - options, _ = parser.parse_args(argv) - - if not all([options.inputs, options.output]): - l.LogError('Must supply --inputs and --output') - return 1 - - try: - pm = ProfileMerger( - options.inputs.split(','), options.output, int(options.chunk_size), - options.merge_program, options.multipliers) - pm.DoMerge() - retval = 0 - except: - retval = 1 - finally: - print 'My work is done...' - return retval - - -if __name__ == '__main__': - retval = Main(sys.argv) - sys.exit(retval) diff --git a/deprecated/fdo_scripts/divide_and_merge_profiles_test.py b/deprecated/fdo_scripts/divide_and_merge_profiles_test.py deleted file mode 100755 index 2bfb5cf1..00000000 --- a/deprecated/fdo_scripts/divide_and_merge_profiles_test.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -import os -import random -import shutil -import tempfile -import unittest - -from cros_utils import command_executer -from cros_utils import misc - - -class DivideAndMergeProfilesTest(unittest.TestCase): - - def tearDown(self): - shutil.rmtree(self._program_dir) - for profile_dir in self._profile_dirs: - shutil.rmtree(profile_dir) - - def setUp(self): - self._ce = command_executer.GetCommandExecuter() - self._program_dir = tempfile.mkdtemp() - self._writeProgram() - self._writeMakefile() - with misc.WorkingDirectory(self._program_dir): - self._ce.RunCommand('make') - num_profile_dirs = 2 - self._profile_dirs = [] - for i in range(num_profile_dirs): - profile_dir = tempfile.mkdtemp() - command = ('GCOV_PREFIX_STRIP=%s GCOV_PREFIX=$(/bin/pwd) ' - ' %s/program' % (profile_dir.count('/'), self._program_dir)) - with misc.WorkingDirectory(profile_dir): - self._ce.RunCommand(command) - self._profile_dirs.append(profile_dir) - self._merge_program = '/home/build/static/projects/crosstool/profile-merge/v14.5/profile_merge.par' - - def _writeMakefile(self): - makefile_contents = """ -CC = gcc - -CFLAGS = -fprofile-generate - -SRCS=$(wildcard *.c) - -OBJS=$(SRCS:.c=.o) - -all: program - -program: $(OBJS) - $(CC) -o $@ $^ $(CFLAGS) - -%.o: %.c - $(CC) -c -o $@ $^ $(CFLAGS)""" - - makefile = os.path.join(self._program_dir, 'Makefile') - with open(makefile, 'w') as f: - print >> f, makefile_contents - - def _writeProgram(self, num_files=100): - for i in range(num_files): - current_file = os.path.join(self._program_dir, '%s.c' % i) - with open(current_file, 'w') as f: - if i != num_files - 1: - print >> f, 'extern void foo%s();' % (i + 1) - print >> f, 'void foo%s(){foo%s();}' % (i, i + 1) - else: - print >> f, "void foo%s(){printf(\"\");}" % i - if i == 0: - print >> f, 'int main(){foo%s(); return 0;}' % i - - def testMerge(self): - reference_output = self._getReferenceOutput() - my_output = self._getMyOutput() - - ret = self._diffOutputs(reference_output, my_output) - shutil.rmtree(my_output) - shutil.rmtree(reference_output) - self.assertTrue(ret == 0) - - def _diffOutputs(self, reference, mine): - command = 'diff -uNr %s %s' % (reference, mine) - return self._ce.RunCommand(command) - - def _getMyOutput(self, args=''): - my_output = tempfile.mkdtemp() - my_merge_program = os.path.join( - os.path.dirname(__file__), 'divide_and_merge_profiles.py') - command = ('python %s --inputs=%s --output=%s ' - '--chunk_size=10 ' - '--merge_program=%s ' - '%s' % (my_merge_program, ','.join(self._profile_dirs), - my_output, self._merge_program, args)) - self._ce.RunCommand(command) - return my_output - - def _getReferenceOutput(self, args=''): - # First do a regular merge. - reference_output = tempfile.mkdtemp() - command = ('%s --inputs=%s --output=%s %s' % - (self._merge_program, ','.join(self._profile_dirs), - reference_output, args)) - self._ce.RunCommand(command) - return reference_output - - def testMergeWithMultipliers(self): - num_profiles = len(self._profile_dirs) - multipliers = [str(random.randint(0, num_profiles)) \ - for _ in range(num_profiles)] - args = '--multipliers=%s' % ','.join(multipliers) - - reference_output = self._getReferenceOutput(args) - my_output = self._getMyOutput(args) - - ret = self._diffOutputs(reference_output, my_output) - - shutil.rmtree(my_output) - shutil.rmtree(reference_output) - self.assertTrue(ret == 0) - - -if __name__ == '__main__': - unittest.main() diff --git a/deprecated/fdo_scripts/profile_cycler.py b/deprecated/fdo_scripts/profile_cycler.py deleted file mode 100755 index 176f2d4a..00000000 --- a/deprecated/fdo_scripts/profile_cycler.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2011 Google Inc. All Rights Reserved. -"""Script to profile a page cycler, and get it back to the host.""" - -import copy -import optparse -import os -import pickle -import re -import sys -import tempfile -import time - -import build_chrome_browser -import cros_login -import lock_machine -import run_tests -from cros_utils import command_executer -from cros_utils import logger -from cros_utils import misc - - -class CyclerProfiler: - REMOTE_TMP_DIR = '/tmp' - - def __init__(self, chromeos_root, board, cycler, profile_dir, remote): - self._chromeos_root = chromeos_root - self._cycler = cycler - self._profile_dir = profile_dir - self._remote = remote - self._board = board - self._ce = command_executer.GetCommandExecuter() - self._l = logger.GetLogger() - - self._gcov_prefix = os.path.join(self.REMOTE_TMP_DIR, self._GetProfileDir()) - - def _GetProfileDir(self): - return misc.GetCtargetFromBoard(self._board, self._chromeos_root) - - def _CopyTestData(self): - page_cycler_dir = os.path.join(self._chromeos_root, 'distfiles', 'target', - 'chrome-src-internal', 'src', 'data', - 'page_cycler') - if not os.path.isdir(page_cycler_dir): - raise RuntimeError('Page cycler dir %s not found!' % page_cycler_dir) - self._ce.CopyFiles(page_cycler_dir, - os.path.join(self.REMOTE_TMP_DIR, 'page_cycler'), - dest_machine=self._remote, - chromeos_root=self._chromeos_root, - recursive=True, - dest_cros=True) - - def _PrepareTestData(self): - # chmod files so everyone can read them. - command = ('cd %s && find page_cycler -type f | xargs chmod a+r' % - self.REMOTE_TMP_DIR) - self._ce.CrosRunCommand(command, - chromeos_root=self._chromeos_root, - machine=self._remote) - command = ('cd %s && find page_cycler -type d | xargs chmod a+rx' % - self.REMOTE_TMP_DIR) - self._ce.CrosRunCommand(command, - chromeos_root=self._chromeos_root, - machine=self._remote) - - def _CopyProfileToHost(self): - dest_dir = os.path.join(self._profile_dir, - os.path.basename(self._gcov_prefix)) - # First remove the dir if it exists already - if os.path.exists(dest_dir): - command = 'rm -rf %s' % dest_dir - self._ce.RunCommand(command) - - # Strip out the initial prefix for the Chrome directory before doing the - # copy. - chrome_dir_prefix = misc.GetChromeSrcDir() - - command = 'mkdir -p %s' % dest_dir - self._ce.RunCommand(command) - self._ce.CopyFiles(self._gcov_prefix, - dest_dir, - src_machine=self._remote, - chromeos_root=self._chromeos_root, - recursive=True, - src_cros=True) - - def _RemoveRemoteProfileDir(self): - command = 'rm -rf %s' % self._gcov_prefix - self._ce.CrosRunCommand(command, - chromeos_root=self._chromeos_root, - machine=self._remote) - - def _LaunchCycler(self, cycler): - command = ( - 'DISPLAY=:0 ' - 'XAUTHORITY=/home/chronos/.Xauthority ' - 'GCOV_PREFIX=%s ' - 'GCOV_PREFIX_STRIP=3 ' - '/opt/google/chrome/chrome ' - '--no-sandbox ' - '--renderer-clean-exit ' - '--user-data-dir=$(mktemp -d) ' - "--url \"file:///%s/page_cycler/%s/start.html?iterations=10&auto=1\" " - '--enable-file-cookies ' - '--no-first-run ' - '--js-flags=expose_gc &' % (self._gcov_prefix, self.REMOTE_TMP_DIR, - cycler)) - - self._ce.CrosRunCommand(command, - chromeos_root=self._chromeos_root, - machine=self._remote, - command_timeout=60) - - def _PkillChrome(self, signal='9'): - command = 'pkill -%s chrome' % signal - self._ce.CrosRunCommand(command, - chromeos_root=self._chromeos_root, - machine=self._remote) - - def DoProfile(self): - # Copy the page cycler data to the remote - self._CopyTestData() - self._PrepareTestData() - self._RemoveRemoteProfileDir() - - for cycler in self._cycler.split(','): - self._ProfileOneCycler(cycler) - - # Copy the profile back - self._CopyProfileToHost() - - def _ProfileOneCycler(self, cycler): - # With aura, all that's needed is a stop/start ui. - self._PkillChrome() - cros_login.RestartUI(self._remote, self._chromeos_root, login=False) - # Run the cycler - self._LaunchCycler(cycler) - self._PkillChrome(signal='INT') - # Let libgcov dump the profile. - # TODO(asharif): There is a race condition here. Fix it later. - time.sleep(30) - - -def Main(argv): - """The main function.""" - # Common initializations - ### command_executer.InitCommandExecuter(True) - command_executer.InitCommandExecuter() - l = logger.GetLogger() - ce = command_executer.GetCommandExecuter() - parser = optparse.OptionParser() - parser.add_option('--cycler', - dest='cycler', - default='alexa_us', - help=('Comma-separated cyclers to profile. ' - 'Example: alexa_us,moz,moz2' - 'Use all to profile all cyclers.')) - parser.add_option('--chromeos_root', - dest='chromeos_root', - default='../../', - help='Output profile directory.') - parser.add_option('--board', - dest='board', - default='x86-zgb', - help='The target board.') - parser.add_option('--remote', - dest='remote', - help=('The remote chromeos machine that' - ' has the profile image.')) - parser.add_option('--profile_dir', - dest='profile_dir', - default='profile_dir', - help='Store profiles in this directory.') - - options, _ = parser.parse_args(argv) - - all_cyclers = ['alexa_us', 'bloat', 'dhtml', 'dom', 'intl1', 'intl2', - 'morejs', 'morejsnp', 'moz', 'moz2'] - - if options.cycler == 'all': - options.cycler = ','.join(all_cyclers) - - try: - cp = CyclerProfiler(options.chromeos_root, options.board, options.cycler, - options.profile_dir, options.remote) - cp.DoProfile() - retval = 0 - except Exception as e: - retval = 1 - print e - finally: - print 'Exiting...' - return retval - - -if __name__ == '__main__': - retval = Main(sys.argv) - sys.exit(retval) diff --git a/deprecated/fdo_scripts/summarize_hot_blocks.py b/deprecated/fdo_scripts/summarize_hot_blocks.py deleted file mode 100644 index 20c07fa4..00000000 --- a/deprecated/fdo_scripts/summarize_hot_blocks.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. -"""Summarize hottest basic blocks found while doing a ChromeOS FDO build. - -Here is an example execution: - - summarize_hot_blocks.py - --data_dir=~/chromeos/chroot/var/cache/chromeos-chrome/ --cutoff=10000 - --output_dir=/home/x/y - -With the cutoff, it will ignore any basic blocks that have a count less -than what is specified (in this example 10000) -The script looks inside the directory (this is typically a directory where -the object files are generated) for files with *.profile and *.optimized -suffixes. To get these, the following flags were added to the compiler -invokation within vanilla_vs_fdo.py in the profile-use phase. - - "-fdump-tree-optimized-blocks-lineno " - "-fdump-ipa-profile-blocks-lineno " - -Here is an example of the *.profile and *.optimized files contents: - -# BLOCK 7 freq:3901 count:60342, starting at line 92 -# PRED: 6 [39.0%] count:60342 (true,exec) - [url_canon_internal.cc : 92:28] MEM[(const char * *)source_6(D) + 16B] = - D.28080_17; - [url_canon_internal.cc : 93:41] MEM[(struct Component *)parsed_4(D) + 16B] = - MEM[(const struct Component &)repl_1(D) + 80]; -# SUCC: 8 [100.0%] count:60342 (fallthru,exec) -# BLOCK 8 freq:10000 count:154667, starting at line 321 -# PRED: 7 [100.0%] count:60342 (fallthru,exec) 6 [61.0%] count:94325 -(false,exec) - [url_canon_internal.cc : 321:51] # DEBUG D#10 => - [googleurl/src/url_canon_internal.cc : 321] &parsed_4(D)->host - -this script finds the blocks with highest count and shows the first line -of each block so that it is easy to identify the origin of the basic block. - -""" - -__author__ = 'llozano@google.com (Luis Lozano)' - -import optparse -import os -import re -import shutil -import sys -import tempfile - -from cros_utils import command_executer - - -# Given a line, check if it has a block count and return it. -# Return -1 if there is no match -def GetBlockCount(line): - match_obj = re.match('.*# BLOCK \d+ .*count:(\d+)', line) - if match_obj: - return int(match_obj.group(1)) - else: - return -1 - - -class Collector(object): - - def __init__(self, data_dir, cutoff, output_dir, tempdir): - self._data_dir = data_dir - self._cutoff = cutoff - self._output_dir = output_dir - self._tempdir = tempdir - self._ce = command_executer.GetCommandExecuter() - - def CollectFileList(self, file_exp, list_file): - command = ("find %s -type f -name '%s' > %s" % - (self._data_dir, file_exp, - os.path.join(self._tempdir, list_file))) - ret = self._ce.RunCommand(command) - if ret: - raise RuntimeError('Failed: %s' % command) - - def SummarizeLines(self, data_file): - sum_lines = [] - search_lno = False - for line in data_file: - count = GetBlockCount(line) - if count != -1: - if count >= self._cutoff: - search_lno = True - sum_line = line.strip() - sum_count = count - # look for a line that starts with line number information - elif search_lno and re.match('^\s*\[.*: \d*:\d*]', line): - search_lno = False - sum_lines.append('%d:%s: %s %s' % - (sum_count, data_file.name, sum_line, line)) - return sum_lines - - # Look for blocks in the data file that have a count larger than the cutoff - # and generate a sorted summary file of the hottest blocks. - def SummarizeFile(self, data_file, sum_file): - with open(data_file, 'r') as f: - sum_lines = self.SummarizeLines(f) - - # sort reverse the list in place by the block count number - sum_lines.sort(key=GetBlockCount, reverse=True) - - with open(sum_file, 'w') as sf: - sf.write(''.join(sum_lines)) - - print 'Generated file Summary: ', sum_file - - # Find hottest blocks in the list of files, generate a sorted summary for - # each file and then do a sorted merge of all the summaries. - def SummarizeList(self, list_file, summary_file): - with open(os.path.join(self._tempdir, list_file)) as f: - sort_list = [] - for file_name in f: - file_name = file_name.strip() - sum_file = '%s.sum' % file_name - sort_list.append('%s%s' % (sum_file, chr(0))) - self.SummarizeFile(file_name, sum_file) - - tmp_list_file = os.path.join(self._tempdir, 'file_list.dat') - with open(tmp_list_file, 'w') as file_list_file: - for x in sort_list: - file_list_file.write(x) - - merge_command = ('sort -nr -t: -k1 --merge --files0-from=%s > %s ' % - (tmp_list_file, summary_file)) - - ret = self._ce.RunCommand(merge_command) - if ret: - raise RuntimeError('Failed: %s' % merge_command) - print 'Generated general summary: ', summary_file - - def SummarizePreOptimized(self, summary_file): - self.CollectFileList('*.profile', 'chrome.profile.list') - self.SummarizeList('chrome.profile.list', - os.path.join(self._output_dir, summary_file)) - - def SummarizeOptimized(self, summary_file): - self.CollectFileList('*.optimized', 'chrome.optimized.list') - self.SummarizeList('chrome.optimized.list', - os.path.join(self._output_dir, summary_file)) - - -def Main(argv): - command_executer.InitCommandExecuter() - usage = ('usage: %prog --data_dir=<dir> --cutoff=<value> ' - '--output_dir=<dir> [--keep_tmp]') - parser = optparse.OptionParser(usage=usage) - parser.add_option('--data_dir', - dest='data_dir', - help=('directory where the FDO (*.profile and ' - '*.optimized) files are located')) - parser.add_option('--cutoff', - dest='cutoff', - help='Minimum count to consider for each basic block') - parser.add_option('--output_dir', - dest='output_dir', - help=('directory where summary data will be generated' - '(pre_optimized.txt, optimized.txt)')) - parser.add_option('--keep_tmp', - action='store_true', - dest='keep_tmp', - default=False, - help=('Keep directory with temporary files' - '(for debugging purposes)')) - options = parser.parse_args(argv)[0] - if not all((options.data_dir, options.cutoff, options.output_dir)): - parser.print_help() - sys.exit(1) - - tempdir = tempfile.mkdtemp() - - co = Collector(options.data_dir, int(options.cutoff), options.output_dir, - tempdir) - co.SummarizePreOptimized('pre_optimized.txt') - co.SummarizeOptimized('optimized.txt') - - if not options.keep_tmp: - shutil.rmtree(tempdir, ignore_errors=True) - - return 0 - - -if __name__ == '__main__': - retval = Main(sys.argv) - sys.exit(retval) diff --git a/deprecated/fdo_scripts/vanilla_vs_fdo.py b/deprecated/fdo_scripts/vanilla_vs_fdo.py deleted file mode 100644 index 6f42839d..00000000 --- a/deprecated/fdo_scripts/vanilla_vs_fdo.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. -"""Script to build chrome with FDO and compare performance against no FDO.""" - -import getpass -import optparse -import os -import sys - -import image_chromeos -import setup_chromeos -from cros_utils import command_executer -from cros_utils import misc -from cros_utils import logger - - -class Patcher(object): - - def __init__(self, dir_to_patch, patch_file): - self._dir_to_patch = dir_to_patch - self._patch_file = patch_file - self._base_patch_command = 'patch -p0 %%s < %s' % patch_file - self._ce = command_executer.GetCommandExecuter() - - def _RunPatchCommand(self, args): - patch_command = self._base_patch_command % args - command = ('cd %s && %s' % (self._dir_to_patch, patch_command)) - return self._ce.RunCommand(command) - - def _ApplyPatch(self, args): - full_args = '%s --dry-run' % args - ret = self._RunPatchCommand(full_args) - if ret: - raise RuntimeError('Patch dry run failed!') - ret = self._RunPatchCommand(args) - if ret: - raise RuntimeError('Patch application failed!') - - def __enter__(self): - self._ApplyPatch('') - - def __exit__(self, type, value, traceback): - self._ApplyPatch('-R') - - -class FDOComparator(object): - - def __init__(self, board, remotes, ebuild_version, plus_pgo, minus_pgo, - update_pgo, chromeos_root): - self._board = board - self._remotes = remotes - self._ebuild_version = ebuild_version - self._remote = remotes.split(',')[0] - self._chromeos_root = chromeos_root - self._profile_dir = 'profile_dir' - self._profile_path = os.path.join(self._chromeos_root, 'src', 'scripts', - os.path.basename(self._profile_dir)) - self._plus_pgo = plus_pgo - self._minus_pgo = minus_pgo - self._update_pgo = update_pgo - - self._ce = command_executer.GetCommandExecuter() - self._l = logger.GetLogger() - - def _CheckoutChromeOS(self): - if not os.path.exists(self._chromeos_root): - setup_chromeos_args = [setup_chromeos.__file__, - '--dir=%s' % self._chromeos_root, '--minilayout'] - setup_chromeos.Main(setup_chromeos_args) - - def _BuildChromeOSUsingBinaries(self): - image_dir = misc.GetImageDir(self._chromeos_root, self._board) - command = 'equery-%s l chromeos' % self._board - ret = self._ce.ChrootRunCommand(self._chromeos_root, command) - if ret: - command = misc.GetSetupBoardCommand(self._board, usepkg=True) - ret = self._ce.ChrootRunCommand(self._chromeos_root, command) - if ret: - raise RuntimeError("Couldn't run setup_board!") - command = misc.GetBuildPackagesCommand(self._board, True) - ret = self._ce.ChrootRunCommand(self._chromeos_root, command) - if ret: - raise RuntimeError("Couldn't run build_packages!") - - def _ReportMismatches(self, build_log): - mismatch_signature = '-Wcoverage-mismatch' - mismatches = build_log.count(mismatch_signature) - self._l.LogOutput('Total mismatches: %s' % mismatches) - stale_files = set([]) - for line in build_log.splitlines(): - if mismatch_signature in line: - filename = line.split(':')[0] - stale_files.add(filename) - self._l.LogOutput('Total stale files: %s' % len(stale_files)) - - def _BuildChromeAndImage(self, - ebuild_version='', - env_dict={}, - cflags='', - cxxflags='', - ldflags='', - label='', - build_image_args=''): - env_string = misc.GetEnvStringFromDict(env_dict) - if not label: - label = ' '.join([env_string, cflags, cxxflags, ldflags, ebuild_version]) - label = label.strip() - label = misc.GetFilenameFromString(label) - if not misc.DoesLabelExist(self._chromeos_root, self._board, label): - build_chrome_browser_args = ['--clean', '--chromeos_root=%s' % - self._chromeos_root, '--board=%s' % - self._board, '--env=%r' % env_string, - '--cflags=%r' % cflags, '--cxxflags=%r' % - cxxflags, '--ldflags=%r' % ldflags, - '--ebuild_version=%s' % ebuild_version, - '--build_image_args=%s' % build_image_args] - - build_chrome_browser = os.path.join( - os.path.dirname(__file__), '..', 'build_chrome_browser.py') - command = 'python %s %s' % (build_chrome_browser, - ' '.join(build_chrome_browser_args)) - ret, out, err = self._ce.RunCommandWOutput(command) - if '-fprofile-use' in cxxflags: - self._ReportMismatches(out) - - if ret: - raise RuntimeError("Couldn't build chrome browser!") - misc.LabelLatestImage(self._chromeos_root, self._board, label) - return label - - def _TestLabels(self, labels): - experiment_file = 'pgo_experiment.txt' - experiment_header = """ - board: %s - remote: %s - """ % (self._board, self._remotes) - experiment_tests = """ - benchmark: desktopui_PyAutoPerfTests { - iterations: 1 - } - """ - - with open(experiment_file, 'w') as f: - print >> f, experiment_header - print >> f, experiment_tests - for label in labels: - # TODO(asharif): Fix crosperf so it accepts labels with symbols - crosperf_label = label - crosperf_label = crosperf_label.replace('-', 'minus') - crosperf_label = crosperf_label.replace('+', 'plus') - experiment_image = """ - %s { - chromeos_image: %s - } - """ % (crosperf_label, os.path.join( - misc.GetImageDir(self._chromeos_root, self._board), label, - 'chromiumos_test_image.bin')) - print >> f, experiment_image - crosperf = os.path.join( - os.path.dirname(__file__), '..', 'crosperf', 'crosperf') - command = '%s %s' % (crosperf, experiment_file) - ret = self._ce.RunCommand(command) - if ret: - raise RuntimeError("Couldn't run crosperf!") - - def _ImageRemote(self, label): - image_path = os.path.join( - misc.GetImageDir(self._chromeos_root, - self._board), label, 'chromiumos_test_image.bin') - image_chromeos_args = [image_chromeos.__file__, '--chromeos_root=%s' % - self._chromeos_root, '--image=%s' % image_path, - '--remote=%s' % self._remote, - '--board=%s' % self._board] - image_chromeos.Main(image_chromeos_args) - - def _ProfileRemote(self): - profile_cycler = os.path.join( - os.path.dirname(__file__), 'profile_cycler.py') - profile_cycler_args = ['--chromeos_root=%s' % self._chromeos_root, - '--cycler=all', '--board=%s' % self._board, - '--profile_dir=%s' % self._profile_path, - '--remote=%s' % self._remote] - command = 'python %s %s' % (profile_cycler, ' '.join(profile_cycler_args)) - ret = self._ce.RunCommand(command) - if ret: - raise RuntimeError("Couldn't profile cycler!") - - def _BuildGenerateImage(self): - # TODO(asharif): add cflags as well. - labels_list = ['fprofile-generate', self._ebuild_version] - label = '_'.join(labels_list) - generate_label = self._BuildChromeAndImage( - env_dict={'USE': 'chrome_internal -pgo pgo_generate'}, - label=label, - ebuild_version=self._ebuild_version, - build_image_args='--rootfs_boost_size=400') - return generate_label - - def _BuildUseImage(self): - ctarget = misc.GetCtargetFromBoard(self._board, self._chromeos_root) - chroot_profile_dir = os.path.join('/home/%s/trunk' % getpass.getuser(), - 'src', 'scripts', self._profile_dir, - ctarget) - cflags = ('-fprofile-use ' - '-fprofile-correction ' - '-Wno-error ' - '-fdump-tree-optimized-blocks-lineno ' - '-fdump-ipa-profile-blocks-lineno ' - '-fno-vpt ' - '-fprofile-dir=%s' % chroot_profile_dir) - labels_list = ['updated_pgo', self._ebuild_version] - label = '_'.join(labels_list) - pgo_use_label = self._BuildChromeAndImage( - env_dict={'USE': 'chrome_internal -pgo'}, - cflags=cflags, - cxxflags=cflags, - ldflags=cflags, - label=label, - ebuild_version=self._ebuild_version) - return pgo_use_label - - def DoAll(self): - self._CheckoutChromeOS() - self._BuildChromeOSUsingBinaries() - labels = [] - - if self._minus_pgo: - minus_pgo = self._BuildChromeAndImage( - env_dict={'USE': 'chrome_internal -pgo'}, - ebuild_version=self._ebuild_version) - labels.append(minus_pgo) - if self._plus_pgo: - plus_pgo = self._BuildChromeAndImage( - env_dict={'USE': 'chrome_internal pgo'}, - ebuild_version=self._ebuild_version) - labels.append(plus_pgo) - - if self._update_pgo: - if not os.path.exists(self._profile_path): - # Build Chrome with -fprofile-generate - generate_label = self._BuildGenerateImage() - # Image to the remote box. - self._ImageRemote(generate_label) - # Profile it using all page cyclers. - self._ProfileRemote() - - # Use the profile directory to rebuild it. - updated_pgo_label = self._BuildUseImage() - labels.append(updated_pgo_label) - - # Run crosperf on all images now. - self._TestLabels(labels) - return 0 - - -def Main(argv): - """The main function.""" - # Common initializations - ### command_executer.InitCommandExecuter(True) - command_executer.InitCommandExecuter() - parser = optparse.OptionParser() - parser.add_option('--remote', - dest='remote', - help='Remote machines to run tests on.') - parser.add_option('--board', - dest='board', - default='x86-zgb', - help='The target board.') - parser.add_option('--ebuild_version', - dest='ebuild_version', - default='', - help='The Chrome ebuild version to use.') - parser.add_option('--plus_pgo', - dest='plus_pgo', - action='store_true', - default=False, - help='Build USE=+pgo.') - parser.add_option('--minus_pgo', - dest='minus_pgo', - action='store_true', - default=False, - help='Build USE=-pgo.') - parser.add_option('--update_pgo', - dest='update_pgo', - action='store_true', - default=False, - help='Update pgo and build Chrome with the update.') - parser.add_option('--chromeos_root', - dest='chromeos_root', - default=False, - help='The chromeos root directory') - options, _ = parser.parse_args(argv) - if not options.board: - print 'Please give a board.' - return 1 - if not options.remote: - print 'Please give at least one remote machine.' - return 1 - if not options.chromeos_root: - print 'Please provide the chromeos root directory.' - return 1 - if not any((options.minus_pgo, options.plus_pgo, options.update_pgo)): - print 'Please provide at least one build option.' - return 1 - fc = FDOComparator(options.board, options.remote, options.ebuild_version, - options.plus_pgo, options.minus_pgo, options.update_pgo, - os.path.expanduser(options.chromeos_root)) - return fc.DoAll() - - -if __name__ == '__main__': - retval = Main(sys.argv) - sys.exit(retval) |