diff options
Diffstat (limited to 'deprecated/fdo_scripts')
| -rwxr-xr-x | deprecated/fdo_scripts/divide_and_merge_profiles.py | 140 | ||||
| -rwxr-xr-x | deprecated/fdo_scripts/divide_and_merge_profiles_test.py | 127 | ||||
| -rwxr-xr-x | deprecated/fdo_scripts/profile_cycler.py | 199 | ||||
| -rw-r--r-- | deprecated/fdo_scripts/summarize_hot_blocks.py | 187 | ||||
| -rw-r--r-- | deprecated/fdo_scripts/vanilla_vs_fdo.py | 312 |
5 files changed, 965 insertions, 0 deletions
diff --git a/deprecated/fdo_scripts/divide_and_merge_profiles.py b/deprecated/fdo_scripts/divide_and_merge_profiles.py new file mode 100755 index 00000000..5108aa6f --- /dev/null +++ b/deprecated/fdo_scripts/divide_and_merge_profiles.py @@ -0,0 +1,140 @@ +#!/usr/bin/python2 +# +# Copyright 2011 Google Inc. All Rights Reserved. +"""Script to divide and merge profiles.""" + +import copy +import optparse +import os +import pickle +import re +import sys +import tempfile + +import build_chrome_browser +import lock_machine +import run_tests +from cros_utils import command_executer +from cros_utils import logger + + +class ProfileMerger: + + def __init__(self, inputs, output, chunk_size, merge_program, multipliers): + self._inputs = inputs + self._output = output + self._chunk_size = chunk_size + self._merge_program = merge_program + self._multipliers = multipliers + self._ce = command_executer.GetCommandExecuter() + self._l = logger.GetLogger() + + def _GetFilesSetForInputDir(self, input_dir): + output_file = tempfile.mktemp() + command = "find %s -name '*.gcda' -o -name '*.imports' > %s" % (input_dir, + output_file) + self._ce.RunCommand(command) + files = open(output_file, 'r').read() + files_set = set([]) + for f in files.splitlines(): + stripped_file = f.replace(input_dir, '', 1) + stripped_file = stripped_file.lstrip('/') + files_set.add(stripped_file) + return files_set + + def _PopulateFilesSet(self): + self._files_set = set([]) + for i in self._inputs: + current_files_set = self._GetFilesSetForInputDir(i) + self._files_set.update(current_files_set) + + def _GetSubset(self): + ret = [] + for i in range(self._chunk_size): + if not self._files_set: + break + ret.append(self._files_set.pop()) + return ret + + def _CopyFilesTree(self, input_dir, files, output_dir): + for f in files: + src_file = os.path.join(input_dir, f) + dst_file = os.path.join(output_dir, f) + if not os.path.isdir(os.path.dirname(dst_file)): + command = 'mkdir -p %s' % os.path.dirname(dst_file) + self._ce.RunCommand(command) + command = 'cp %s %s' % (src_file, dst_file) + self._ce.RunCommand(command) + + def _DoChunkMerge(self, current_files): + temp_dirs = [] + for i in self._inputs: + temp_dir = tempfile.mkdtemp() + temp_dirs.append(temp_dir) + self._CopyFilesTree(i, current_files, temp_dir) + # Now do the merge. + command = ('%s --inputs=%s --output=%s' % + (self._merge_program, ','.join(temp_dirs), self._output)) + if self._multipliers: + command = ('%s --multipliers=%s' % (command, self._multipliers)) + ret = self._ce.RunCommand(command) + assert ret == 0, '%s command failed!' % command + for temp_dir in temp_dirs: + command = 'rm -rf %s' % temp_dir + self._ce.RunCommand(command) + + def DoMerge(self): + self._PopulateFilesSet() + while True: + current_files = self._GetSubset() + if not current_files: + break + self._DoChunkMerge(current_files) + + +def Main(argv): + """The main function.""" + # Common initializations + ### command_executer.InitCommandExecuter(True) + command_executer.InitCommandExecuter() + l = logger.GetLogger() + ce = command_executer.GetCommandExecuter() + parser = optparse.OptionParser() + parser.add_option('--inputs', + dest='inputs', + help='Comma-separated input profile directories to merge.') + parser.add_option('--output', dest='output', help='Output profile directory.') + parser.add_option('--chunk_size', + dest='chunk_size', + default='50', + help='Chunk size to divide up the profiles into.') + parser.add_option('--merge_program', + dest='merge_program', + default='/home/xur/bin/profile_merge_v15.par', + help='Merge program to use to do the actual merge.') + parser.add_option('--multipliers', + dest='multipliers', + help='multipliers to use when merging. (optional)') + + options, _ = parser.parse_args(argv) + + if not all([options.inputs, options.output]): + l.LogError('Must supply --inputs and --output') + return 1 + + try: + pm = ProfileMerger( + options.inputs.split(','), options.output, int(options.chunk_size), + options.merge_program, options.multipliers) + pm.DoMerge() + retval = 0 + except: + retval = 1 + finally: + print 'My work is done...' + return retval + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/deprecated/fdo_scripts/divide_and_merge_profiles_test.py b/deprecated/fdo_scripts/divide_and_merge_profiles_test.py new file mode 100755 index 00000000..2bfb5cf1 --- /dev/null +++ b/deprecated/fdo_scripts/divide_and_merge_profiles_test.py @@ -0,0 +1,127 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import os +import random +import shutil +import tempfile +import unittest + +from cros_utils import command_executer +from cros_utils import misc + + +class DivideAndMergeProfilesTest(unittest.TestCase): + + def tearDown(self): + shutil.rmtree(self._program_dir) + for profile_dir in self._profile_dirs: + shutil.rmtree(profile_dir) + + def setUp(self): + self._ce = command_executer.GetCommandExecuter() + self._program_dir = tempfile.mkdtemp() + self._writeProgram() + self._writeMakefile() + with misc.WorkingDirectory(self._program_dir): + self._ce.RunCommand('make') + num_profile_dirs = 2 + self._profile_dirs = [] + for i in range(num_profile_dirs): + profile_dir = tempfile.mkdtemp() + command = ('GCOV_PREFIX_STRIP=%s GCOV_PREFIX=$(/bin/pwd) ' + ' %s/program' % (profile_dir.count('/'), self._program_dir)) + with misc.WorkingDirectory(profile_dir): + self._ce.RunCommand(command) + self._profile_dirs.append(profile_dir) + self._merge_program = '/home/build/static/projects/crosstool/profile-merge/v14.5/profile_merge.par' + + def _writeMakefile(self): + makefile_contents = """ +CC = gcc + +CFLAGS = -fprofile-generate + +SRCS=$(wildcard *.c) + +OBJS=$(SRCS:.c=.o) + +all: program + +program: $(OBJS) + $(CC) -o $@ $^ $(CFLAGS) + +%.o: %.c + $(CC) -c -o $@ $^ $(CFLAGS)""" + + makefile = os.path.join(self._program_dir, 'Makefile') + with open(makefile, 'w') as f: + print >> f, makefile_contents + + def _writeProgram(self, num_files=100): + for i in range(num_files): + current_file = os.path.join(self._program_dir, '%s.c' % i) + with open(current_file, 'w') as f: + if i != num_files - 1: + print >> f, 'extern void foo%s();' % (i + 1) + print >> f, 'void foo%s(){foo%s();}' % (i, i + 1) + else: + print >> f, "void foo%s(){printf(\"\");}" % i + if i == 0: + print >> f, 'int main(){foo%s(); return 0;}' % i + + def testMerge(self): + reference_output = self._getReferenceOutput() + my_output = self._getMyOutput() + + ret = self._diffOutputs(reference_output, my_output) + shutil.rmtree(my_output) + shutil.rmtree(reference_output) + self.assertTrue(ret == 0) + + def _diffOutputs(self, reference, mine): + command = 'diff -uNr %s %s' % (reference, mine) + return self._ce.RunCommand(command) + + def _getMyOutput(self, args=''): + my_output = tempfile.mkdtemp() + my_merge_program = os.path.join( + os.path.dirname(__file__), 'divide_and_merge_profiles.py') + command = ('python %s --inputs=%s --output=%s ' + '--chunk_size=10 ' + '--merge_program=%s ' + '%s' % (my_merge_program, ','.join(self._profile_dirs), + my_output, self._merge_program, args)) + self._ce.RunCommand(command) + return my_output + + def _getReferenceOutput(self, args=''): + # First do a regular merge. + reference_output = tempfile.mkdtemp() + command = ('%s --inputs=%s --output=%s %s' % + (self._merge_program, ','.join(self._profile_dirs), + reference_output, args)) + self._ce.RunCommand(command) + return reference_output + + def testMergeWithMultipliers(self): + num_profiles = len(self._profile_dirs) + multipliers = [str(random.randint(0, num_profiles)) \ + for _ in range(num_profiles)] + args = '--multipliers=%s' % ','.join(multipliers) + + reference_output = self._getReferenceOutput(args) + my_output = self._getMyOutput(args) + + ret = self._diffOutputs(reference_output, my_output) + + shutil.rmtree(my_output) + shutil.rmtree(reference_output) + self.assertTrue(ret == 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/deprecated/fdo_scripts/profile_cycler.py b/deprecated/fdo_scripts/profile_cycler.py new file mode 100755 index 00000000..176f2d4a --- /dev/null +++ b/deprecated/fdo_scripts/profile_cycler.py @@ -0,0 +1,199 @@ +#!/usr/bin/python2 +# +# Copyright 2011 Google Inc. All Rights Reserved. +"""Script to profile a page cycler, and get it back to the host.""" + +import copy +import optparse +import os +import pickle +import re +import sys +import tempfile +import time + +import build_chrome_browser +import cros_login +import lock_machine +import run_tests +from cros_utils import command_executer +from cros_utils import logger +from cros_utils import misc + + +class CyclerProfiler: + REMOTE_TMP_DIR = '/tmp' + + def __init__(self, chromeos_root, board, cycler, profile_dir, remote): + self._chromeos_root = chromeos_root + self._cycler = cycler + self._profile_dir = profile_dir + self._remote = remote + self._board = board + self._ce = command_executer.GetCommandExecuter() + self._l = logger.GetLogger() + + self._gcov_prefix = os.path.join(self.REMOTE_TMP_DIR, self._GetProfileDir()) + + def _GetProfileDir(self): + return misc.GetCtargetFromBoard(self._board, self._chromeos_root) + + def _CopyTestData(self): + page_cycler_dir = os.path.join(self._chromeos_root, 'distfiles', 'target', + 'chrome-src-internal', 'src', 'data', + 'page_cycler') + if not os.path.isdir(page_cycler_dir): + raise RuntimeError('Page cycler dir %s not found!' % page_cycler_dir) + self._ce.CopyFiles(page_cycler_dir, + os.path.join(self.REMOTE_TMP_DIR, 'page_cycler'), + dest_machine=self._remote, + chromeos_root=self._chromeos_root, + recursive=True, + dest_cros=True) + + def _PrepareTestData(self): + # chmod files so everyone can read them. + command = ('cd %s && find page_cycler -type f | xargs chmod a+r' % + self.REMOTE_TMP_DIR) + self._ce.CrosRunCommand(command, + chromeos_root=self._chromeos_root, + machine=self._remote) + command = ('cd %s && find page_cycler -type d | xargs chmod a+rx' % + self.REMOTE_TMP_DIR) + self._ce.CrosRunCommand(command, + chromeos_root=self._chromeos_root, + machine=self._remote) + + def _CopyProfileToHost(self): + dest_dir = os.path.join(self._profile_dir, + os.path.basename(self._gcov_prefix)) + # First remove the dir if it exists already + if os.path.exists(dest_dir): + command = 'rm -rf %s' % dest_dir + self._ce.RunCommand(command) + + # Strip out the initial prefix for the Chrome directory before doing the + # copy. + chrome_dir_prefix = misc.GetChromeSrcDir() + + command = 'mkdir -p %s' % dest_dir + self._ce.RunCommand(command) + self._ce.CopyFiles(self._gcov_prefix, + dest_dir, + src_machine=self._remote, + chromeos_root=self._chromeos_root, + recursive=True, + src_cros=True) + + def _RemoveRemoteProfileDir(self): + command = 'rm -rf %s' % self._gcov_prefix + self._ce.CrosRunCommand(command, + chromeos_root=self._chromeos_root, + machine=self._remote) + + def _LaunchCycler(self, cycler): + command = ( + 'DISPLAY=:0 ' + 'XAUTHORITY=/home/chronos/.Xauthority ' + 'GCOV_PREFIX=%s ' + 'GCOV_PREFIX_STRIP=3 ' + '/opt/google/chrome/chrome ' + '--no-sandbox ' + '--renderer-clean-exit ' + '--user-data-dir=$(mktemp -d) ' + "--url \"file:///%s/page_cycler/%s/start.html?iterations=10&auto=1\" " + '--enable-file-cookies ' + '--no-first-run ' + '--js-flags=expose_gc &' % (self._gcov_prefix, self.REMOTE_TMP_DIR, + cycler)) + + self._ce.CrosRunCommand(command, + chromeos_root=self._chromeos_root, + machine=self._remote, + command_timeout=60) + + def _PkillChrome(self, signal='9'): + command = 'pkill -%s chrome' % signal + self._ce.CrosRunCommand(command, + chromeos_root=self._chromeos_root, + machine=self._remote) + + def DoProfile(self): + # Copy the page cycler data to the remote + self._CopyTestData() + self._PrepareTestData() + self._RemoveRemoteProfileDir() + + for cycler in self._cycler.split(','): + self._ProfileOneCycler(cycler) + + # Copy the profile back + self._CopyProfileToHost() + + def _ProfileOneCycler(self, cycler): + # With aura, all that's needed is a stop/start ui. + self._PkillChrome() + cros_login.RestartUI(self._remote, self._chromeos_root, login=False) + # Run the cycler + self._LaunchCycler(cycler) + self._PkillChrome(signal='INT') + # Let libgcov dump the profile. + # TODO(asharif): There is a race condition here. Fix it later. + time.sleep(30) + + +def Main(argv): + """The main function.""" + # Common initializations + ### command_executer.InitCommandExecuter(True) + command_executer.InitCommandExecuter() + l = logger.GetLogger() + ce = command_executer.GetCommandExecuter() + parser = optparse.OptionParser() + parser.add_option('--cycler', + dest='cycler', + default='alexa_us', + help=('Comma-separated cyclers to profile. ' + 'Example: alexa_us,moz,moz2' + 'Use all to profile all cyclers.')) + parser.add_option('--chromeos_root', + dest='chromeos_root', + default='../../', + help='Output profile directory.') + parser.add_option('--board', + dest='board', + default='x86-zgb', + help='The target board.') + parser.add_option('--remote', + dest='remote', + help=('The remote chromeos machine that' + ' has the profile image.')) + parser.add_option('--profile_dir', + dest='profile_dir', + default='profile_dir', + help='Store profiles in this directory.') + + options, _ = parser.parse_args(argv) + + all_cyclers = ['alexa_us', 'bloat', 'dhtml', 'dom', 'intl1', 'intl2', + 'morejs', 'morejsnp', 'moz', 'moz2'] + + if options.cycler == 'all': + options.cycler = ','.join(all_cyclers) + + try: + cp = CyclerProfiler(options.chromeos_root, options.board, options.cycler, + options.profile_dir, options.remote) + cp.DoProfile() + retval = 0 + except Exception as e: + retval = 1 + print e + finally: + print 'Exiting...' + return retval + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/deprecated/fdo_scripts/summarize_hot_blocks.py b/deprecated/fdo_scripts/summarize_hot_blocks.py new file mode 100644 index 00000000..20c07fa4 --- /dev/null +++ b/deprecated/fdo_scripts/summarize_hot_blocks.py @@ -0,0 +1,187 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +"""Summarize hottest basic blocks found while doing a ChromeOS FDO build. + +Here is an example execution: + + summarize_hot_blocks.py + --data_dir=~/chromeos/chroot/var/cache/chromeos-chrome/ --cutoff=10000 + --output_dir=/home/x/y + +With the cutoff, it will ignore any basic blocks that have a count less +than what is specified (in this example 10000) +The script looks inside the directory (this is typically a directory where +the object files are generated) for files with *.profile and *.optimized +suffixes. To get these, the following flags were added to the compiler +invokation within vanilla_vs_fdo.py in the profile-use phase. + + "-fdump-tree-optimized-blocks-lineno " + "-fdump-ipa-profile-blocks-lineno " + +Here is an example of the *.profile and *.optimized files contents: + +# BLOCK 7 freq:3901 count:60342, starting at line 92 +# PRED: 6 [39.0%] count:60342 (true,exec) + [url_canon_internal.cc : 92:28] MEM[(const char * *)source_6(D) + 16B] = + D.28080_17; + [url_canon_internal.cc : 93:41] MEM[(struct Component *)parsed_4(D) + 16B] = + MEM[(const struct Component &)repl_1(D) + 80]; +# SUCC: 8 [100.0%] count:60342 (fallthru,exec) +# BLOCK 8 freq:10000 count:154667, starting at line 321 +# PRED: 7 [100.0%] count:60342 (fallthru,exec) 6 [61.0%] count:94325 +(false,exec) + [url_canon_internal.cc : 321:51] # DEBUG D#10 => + [googleurl/src/url_canon_internal.cc : 321] &parsed_4(D)->host + +this script finds the blocks with highest count and shows the first line +of each block so that it is easy to identify the origin of the basic block. + +""" + +__author__ = 'llozano@google.com (Luis Lozano)' + +import optparse +import os +import re +import shutil +import sys +import tempfile + +from cros_utils import command_executer + + +# Given a line, check if it has a block count and return it. +# Return -1 if there is no match +def GetBlockCount(line): + match_obj = re.match('.*# BLOCK \d+ .*count:(\d+)', line) + if match_obj: + return int(match_obj.group(1)) + else: + return -1 + + +class Collector(object): + + def __init__(self, data_dir, cutoff, output_dir, tempdir): + self._data_dir = data_dir + self._cutoff = cutoff + self._output_dir = output_dir + self._tempdir = tempdir + self._ce = command_executer.GetCommandExecuter() + + def CollectFileList(self, file_exp, list_file): + command = ("find %s -type f -name '%s' > %s" % + (self._data_dir, file_exp, + os.path.join(self._tempdir, list_file))) + ret = self._ce.RunCommand(command) + if ret: + raise RuntimeError('Failed: %s' % command) + + def SummarizeLines(self, data_file): + sum_lines = [] + search_lno = False + for line in data_file: + count = GetBlockCount(line) + if count != -1: + if count >= self._cutoff: + search_lno = True + sum_line = line.strip() + sum_count = count + # look for a line that starts with line number information + elif search_lno and re.match('^\s*\[.*: \d*:\d*]', line): + search_lno = False + sum_lines.append('%d:%s: %s %s' % + (sum_count, data_file.name, sum_line, line)) + return sum_lines + + # Look for blocks in the data file that have a count larger than the cutoff + # and generate a sorted summary file of the hottest blocks. + def SummarizeFile(self, data_file, sum_file): + with open(data_file, 'r') as f: + sum_lines = self.SummarizeLines(f) + + # sort reverse the list in place by the block count number + sum_lines.sort(key=GetBlockCount, reverse=True) + + with open(sum_file, 'w') as sf: + sf.write(''.join(sum_lines)) + + print 'Generated file Summary: ', sum_file + + # Find hottest blocks in the list of files, generate a sorted summary for + # each file and then do a sorted merge of all the summaries. + def SummarizeList(self, list_file, summary_file): + with open(os.path.join(self._tempdir, list_file)) as f: + sort_list = [] + for file_name in f: + file_name = file_name.strip() + sum_file = '%s.sum' % file_name + sort_list.append('%s%s' % (sum_file, chr(0))) + self.SummarizeFile(file_name, sum_file) + + tmp_list_file = os.path.join(self._tempdir, 'file_list.dat') + with open(tmp_list_file, 'w') as file_list_file: + for x in sort_list: + file_list_file.write(x) + + merge_command = ('sort -nr -t: -k1 --merge --files0-from=%s > %s ' % + (tmp_list_file, summary_file)) + + ret = self._ce.RunCommand(merge_command) + if ret: + raise RuntimeError('Failed: %s' % merge_command) + print 'Generated general summary: ', summary_file + + def SummarizePreOptimized(self, summary_file): + self.CollectFileList('*.profile', 'chrome.profile.list') + self.SummarizeList('chrome.profile.list', + os.path.join(self._output_dir, summary_file)) + + def SummarizeOptimized(self, summary_file): + self.CollectFileList('*.optimized', 'chrome.optimized.list') + self.SummarizeList('chrome.optimized.list', + os.path.join(self._output_dir, summary_file)) + + +def Main(argv): + command_executer.InitCommandExecuter() + usage = ('usage: %prog --data_dir=<dir> --cutoff=<value> ' + '--output_dir=<dir> [--keep_tmp]') + parser = optparse.OptionParser(usage=usage) + parser.add_option('--data_dir', + dest='data_dir', + help=('directory where the FDO (*.profile and ' + '*.optimized) files are located')) + parser.add_option('--cutoff', + dest='cutoff', + help='Minimum count to consider for each basic block') + parser.add_option('--output_dir', + dest='output_dir', + help=('directory where summary data will be generated' + '(pre_optimized.txt, optimized.txt)')) + parser.add_option('--keep_tmp', + action='store_true', + dest='keep_tmp', + default=False, + help=('Keep directory with temporary files' + '(for debugging purposes)')) + options = parser.parse_args(argv)[0] + if not all((options.data_dir, options.cutoff, options.output_dir)): + parser.print_help() + sys.exit(1) + + tempdir = tempfile.mkdtemp() + + co = Collector(options.data_dir, int(options.cutoff), options.output_dir, + tempdir) + co.SummarizePreOptimized('pre_optimized.txt') + co.SummarizeOptimized('optimized.txt') + + if not options.keep_tmp: + shutil.rmtree(tempdir, ignore_errors=True) + + return 0 + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/deprecated/fdo_scripts/vanilla_vs_fdo.py b/deprecated/fdo_scripts/vanilla_vs_fdo.py new file mode 100644 index 00000000..6f42839d --- /dev/null +++ b/deprecated/fdo_scripts/vanilla_vs_fdo.py @@ -0,0 +1,312 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +"""Script to build chrome with FDO and compare performance against no FDO.""" + +import getpass +import optparse +import os +import sys + +import image_chromeos +import setup_chromeos +from cros_utils import command_executer +from cros_utils import misc +from cros_utils import logger + + +class Patcher(object): + + def __init__(self, dir_to_patch, patch_file): + self._dir_to_patch = dir_to_patch + self._patch_file = patch_file + self._base_patch_command = 'patch -p0 %%s < %s' % patch_file + self._ce = command_executer.GetCommandExecuter() + + def _RunPatchCommand(self, args): + patch_command = self._base_patch_command % args + command = ('cd %s && %s' % (self._dir_to_patch, patch_command)) + return self._ce.RunCommand(command) + + def _ApplyPatch(self, args): + full_args = '%s --dry-run' % args + ret = self._RunPatchCommand(full_args) + if ret: + raise RuntimeError('Patch dry run failed!') + ret = self._RunPatchCommand(args) + if ret: + raise RuntimeError('Patch application failed!') + + def __enter__(self): + self._ApplyPatch('') + + def __exit__(self, type, value, traceback): + self._ApplyPatch('-R') + + +class FDOComparator(object): + + def __init__(self, board, remotes, ebuild_version, plus_pgo, minus_pgo, + update_pgo, chromeos_root): + self._board = board + self._remotes = remotes + self._ebuild_version = ebuild_version + self._remote = remotes.split(',')[0] + self._chromeos_root = chromeos_root + self._profile_dir = 'profile_dir' + self._profile_path = os.path.join(self._chromeos_root, 'src', 'scripts', + os.path.basename(self._profile_dir)) + self._plus_pgo = plus_pgo + self._minus_pgo = minus_pgo + self._update_pgo = update_pgo + + self._ce = command_executer.GetCommandExecuter() + self._l = logger.GetLogger() + + def _CheckoutChromeOS(self): + if not os.path.exists(self._chromeos_root): + setup_chromeos_args = [setup_chromeos.__file__, + '--dir=%s' % self._chromeos_root, '--minilayout'] + setup_chromeos.Main(setup_chromeos_args) + + def _BuildChromeOSUsingBinaries(self): + image_dir = misc.GetImageDir(self._chromeos_root, self._board) + command = 'equery-%s l chromeos' % self._board + ret = self._ce.ChrootRunCommand(self._chromeos_root, command) + if ret: + command = misc.GetSetupBoardCommand(self._board, usepkg=True) + ret = self._ce.ChrootRunCommand(self._chromeos_root, command) + if ret: + raise RuntimeError("Couldn't run setup_board!") + command = misc.GetBuildPackagesCommand(self._board, True) + ret = self._ce.ChrootRunCommand(self._chromeos_root, command) + if ret: + raise RuntimeError("Couldn't run build_packages!") + + def _ReportMismatches(self, build_log): + mismatch_signature = '-Wcoverage-mismatch' + mismatches = build_log.count(mismatch_signature) + self._l.LogOutput('Total mismatches: %s' % mismatches) + stale_files = set([]) + for line in build_log.splitlines(): + if mismatch_signature in line: + filename = line.split(':')[0] + stale_files.add(filename) + self._l.LogOutput('Total stale files: %s' % len(stale_files)) + + def _BuildChromeAndImage(self, + ebuild_version='', + env_dict={}, + cflags='', + cxxflags='', + ldflags='', + label='', + build_image_args=''): + env_string = misc.GetEnvStringFromDict(env_dict) + if not label: + label = ' '.join([env_string, cflags, cxxflags, ldflags, ebuild_version]) + label = label.strip() + label = misc.GetFilenameFromString(label) + if not misc.DoesLabelExist(self._chromeos_root, self._board, label): + build_chrome_browser_args = ['--clean', '--chromeos_root=%s' % + self._chromeos_root, '--board=%s' % + self._board, '--env=%r' % env_string, + '--cflags=%r' % cflags, '--cxxflags=%r' % + cxxflags, '--ldflags=%r' % ldflags, + '--ebuild_version=%s' % ebuild_version, + '--build_image_args=%s' % build_image_args] + + build_chrome_browser = os.path.join( + os.path.dirname(__file__), '..', 'build_chrome_browser.py') + command = 'python %s %s' % (build_chrome_browser, + ' '.join(build_chrome_browser_args)) + ret, out, err = self._ce.RunCommandWOutput(command) + if '-fprofile-use' in cxxflags: + self._ReportMismatches(out) + + if ret: + raise RuntimeError("Couldn't build chrome browser!") + misc.LabelLatestImage(self._chromeos_root, self._board, label) + return label + + def _TestLabels(self, labels): + experiment_file = 'pgo_experiment.txt' + experiment_header = """ + board: %s + remote: %s + """ % (self._board, self._remotes) + experiment_tests = """ + benchmark: desktopui_PyAutoPerfTests { + iterations: 1 + } + """ + + with open(experiment_file, 'w') as f: + print >> f, experiment_header + print >> f, experiment_tests + for label in labels: + # TODO(asharif): Fix crosperf so it accepts labels with symbols + crosperf_label = label + crosperf_label = crosperf_label.replace('-', 'minus') + crosperf_label = crosperf_label.replace('+', 'plus') + experiment_image = """ + %s { + chromeos_image: %s + } + """ % (crosperf_label, os.path.join( + misc.GetImageDir(self._chromeos_root, self._board), label, + 'chromiumos_test_image.bin')) + print >> f, experiment_image + crosperf = os.path.join( + os.path.dirname(__file__), '..', 'crosperf', 'crosperf') + command = '%s %s' % (crosperf, experiment_file) + ret = self._ce.RunCommand(command) + if ret: + raise RuntimeError("Couldn't run crosperf!") + + def _ImageRemote(self, label): + image_path = os.path.join( + misc.GetImageDir(self._chromeos_root, + self._board), label, 'chromiumos_test_image.bin') + image_chromeos_args = [image_chromeos.__file__, '--chromeos_root=%s' % + self._chromeos_root, '--image=%s' % image_path, + '--remote=%s' % self._remote, + '--board=%s' % self._board] + image_chromeos.Main(image_chromeos_args) + + def _ProfileRemote(self): + profile_cycler = os.path.join( + os.path.dirname(__file__), 'profile_cycler.py') + profile_cycler_args = ['--chromeos_root=%s' % self._chromeos_root, + '--cycler=all', '--board=%s' % self._board, + '--profile_dir=%s' % self._profile_path, + '--remote=%s' % self._remote] + command = 'python %s %s' % (profile_cycler, ' '.join(profile_cycler_args)) + ret = self._ce.RunCommand(command) + if ret: + raise RuntimeError("Couldn't profile cycler!") + + def _BuildGenerateImage(self): + # TODO(asharif): add cflags as well. + labels_list = ['fprofile-generate', self._ebuild_version] + label = '_'.join(labels_list) + generate_label = self._BuildChromeAndImage( + env_dict={'USE': 'chrome_internal -pgo pgo_generate'}, + label=label, + ebuild_version=self._ebuild_version, + build_image_args='--rootfs_boost_size=400') + return generate_label + + def _BuildUseImage(self): + ctarget = misc.GetCtargetFromBoard(self._board, self._chromeos_root) + chroot_profile_dir = os.path.join('/home/%s/trunk' % getpass.getuser(), + 'src', 'scripts', self._profile_dir, + ctarget) + cflags = ('-fprofile-use ' + '-fprofile-correction ' + '-Wno-error ' + '-fdump-tree-optimized-blocks-lineno ' + '-fdump-ipa-profile-blocks-lineno ' + '-fno-vpt ' + '-fprofile-dir=%s' % chroot_profile_dir) + labels_list = ['updated_pgo', self._ebuild_version] + label = '_'.join(labels_list) + pgo_use_label = self._BuildChromeAndImage( + env_dict={'USE': 'chrome_internal -pgo'}, + cflags=cflags, + cxxflags=cflags, + ldflags=cflags, + label=label, + ebuild_version=self._ebuild_version) + return pgo_use_label + + def DoAll(self): + self._CheckoutChromeOS() + self._BuildChromeOSUsingBinaries() + labels = [] + + if self._minus_pgo: + minus_pgo = self._BuildChromeAndImage( + env_dict={'USE': 'chrome_internal -pgo'}, + ebuild_version=self._ebuild_version) + labels.append(minus_pgo) + if self._plus_pgo: + plus_pgo = self._BuildChromeAndImage( + env_dict={'USE': 'chrome_internal pgo'}, + ebuild_version=self._ebuild_version) + labels.append(plus_pgo) + + if self._update_pgo: + if not os.path.exists(self._profile_path): + # Build Chrome with -fprofile-generate + generate_label = self._BuildGenerateImage() + # Image to the remote box. + self._ImageRemote(generate_label) + # Profile it using all page cyclers. + self._ProfileRemote() + + # Use the profile directory to rebuild it. + updated_pgo_label = self._BuildUseImage() + labels.append(updated_pgo_label) + + # Run crosperf on all images now. + self._TestLabels(labels) + return 0 + + +def Main(argv): + """The main function.""" + # Common initializations + ### command_executer.InitCommandExecuter(True) + command_executer.InitCommandExecuter() + parser = optparse.OptionParser() + parser.add_option('--remote', + dest='remote', + help='Remote machines to run tests on.') + parser.add_option('--board', + dest='board', + default='x86-zgb', + help='The target board.') + parser.add_option('--ebuild_version', + dest='ebuild_version', + default='', + help='The Chrome ebuild version to use.') + parser.add_option('--plus_pgo', + dest='plus_pgo', + action='store_true', + default=False, + help='Build USE=+pgo.') + parser.add_option('--minus_pgo', + dest='minus_pgo', + action='store_true', + default=False, + help='Build USE=-pgo.') + parser.add_option('--update_pgo', + dest='update_pgo', + action='store_true', + default=False, + help='Update pgo and build Chrome with the update.') + parser.add_option('--chromeos_root', + dest='chromeos_root', + default=False, + help='The chromeos root directory') + options, _ = parser.parse_args(argv) + if not options.board: + print 'Please give a board.' + return 1 + if not options.remote: + print 'Please give at least one remote machine.' + return 1 + if not options.chromeos_root: + print 'Please provide the chromeos root directory.' + return 1 + if not any((options.minus_pgo, options.plus_pgo, options.update_pgo)): + print 'Please provide at least one build option.' + return 1 + fc = FDOComparator(options.board, options.remote, options.ebuild_version, + options.plus_pgo, options.minus_pgo, options.update_pgo, + os.path.expanduser(options.chromeos_root)) + return fc.DoAll() + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) |