Cleaned up directory after copy of tools from perforce directory

Got rid of stale copies of some tools like "crosperf" and moved all
files under v14 directory (that came from perforce) into the top
directory.

BUG=None
TEST=None

Change-Id: I408d17a36ceb00e74db71403d2351fd466a14f8e
Reviewed-on: https://gerrit-int.chromium.org/33887
Tested-by: Luis Lozano <llozano@chromium.org>
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Commit-Queue: Luis Lozano <llozano@chromium.org>

5 files changed, 1000 insertions, 0 deletions

diff --git a/fdo_scripts/divide_and_merge_profiles.py b/fdo_scripts/divide_and_merge_profiles.py
new file mode 100755
index 00000000..c75e353a
--- /dev/null
+++ b/fdo_scripts/divide_and_merge_profiles.py
@@ -0,0 +1,145 @@
+#!/usr/bin/python2.6
+#
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Script to divide and merge profiles."""
+
+import copy
+import optparse
+import os
+import pickle
+import re
+import sys
+import tempfile
+
+import build_chrome_browser
+import lock_machine
+import run_tests
+from utils import command_executer
+from utils import logger
+
+
+class ProfileMerger:
+  def __init__(self, inputs, output, chunk_size, merge_program, multipliers):
+    self._inputs = inputs
+    self._output = output
+    self._chunk_size = chunk_size
+    self._merge_program = merge_program
+    self._multipliers = multipliers
+    self._ce = command_executer.GetCommandExecuter()
+    self._l = logger.GetLogger()
+
+  def _GetFilesSetForInputDir(self, input_dir):
+    output_file = tempfile.mktemp()
+    command = "find %s -name '*.gcda' -o -name '*.imports' > %s" % (input_dir, output_file)
+    self._ce.RunCommand(command)
+    files = open(output_file, "r").read()
+    files_set = set([])
+    for f in files.splitlines():
+      stripped_file = f.replace(input_dir, "", 1)
+      stripped_file = stripped_file.lstrip("/")
+      files_set.add(stripped_file)
+    return files_set
+
+  def _PopulateFilesSet(self):
+    self._files_set = set([])
+    for i in self._inputs:
+      current_files_set = self._GetFilesSetForInputDir(i)
+      self._files_set.update(current_files_set)
+
+  def _GetSubset(self):
+    ret = []
+    for i in range(self._chunk_size):
+      if not self._files_set:
+        break
+      ret.append(self._files_set.pop())
+    return ret
+
+  def _CopyFilesTree(self, input_dir, files, output_dir):
+    for f in files:
+      src_file = os.path.join(input_dir, f)
+      dst_file = os.path.join(output_dir, f)
+      if not os.path.isdir(os.path.dirname(dst_file)):
+        command = "mkdir -p %s" % os.path.dirname(dst_file)
+        self._ce.RunCommand(command)
+      command = "cp %s %s" % (src_file, dst_file)
+      self._ce.RunCommand(command)
+
+  def _DoChunkMerge(self, current_files):
+    temp_dirs = []
+    for i in self._inputs:
+      temp_dir = tempfile.mkdtemp()
+      temp_dirs.append(temp_dir)
+      self._CopyFilesTree(i, current_files, temp_dir)
+    # Now do the merge.
+    command = ("%s --inputs=%s --output=%s" %
+               (self._merge_program,
+                ",".join(temp_dirs),
+                self._output))
+    if self._multipliers:
+      command = ("%s --multipliers=%s" %
+                 (command, self._multipliers))
+    ret = self._ce.RunCommand(command)
+    assert ret == 0, "%s command failed!" % command
+    for temp_dir in temp_dirs:
+      command = "rm -rf %s" % temp_dir
+      self._ce.RunCommand(command)
+
+  def DoMerge(self):
+    self._PopulateFilesSet()
+    while True:
+      current_files = self._GetSubset()
+      if not current_files:
+        break
+      self._DoChunkMerge(current_files)
+
+
+def Main(argv):
+  """The main function."""
+  # Common initializations
+###  command_executer.InitCommandExecuter(True)
+  command_executer.InitCommandExecuter()
+  l = logger.GetLogger()
+  ce = command_executer.GetCommandExecuter()
+  parser = optparse.OptionParser()
+  parser.add_option("--inputs",
+                    dest="inputs",
+                    help="Comma-separated input profile directories to merge.")
+  parser.add_option("--output",
+                    dest="output",
+                    help="Output profile directory.")
+  parser.add_option("--chunk_size",
+                    dest="chunk_size",
+                    default="50",
+                    help="Chunk size to divide up the profiles into.")
+  parser.add_option("--merge_program",
+                    dest="merge_program",
+                    default="/home/xur/bin/profile_merge_v15.par",
+                    help="Merge program to use to do the actual merge.")
+  parser.add_option("--multipliers",
+                    dest="multipliers",
+                    help="multipliers to use when merging. (optional)")
+
+  options, _ = parser.parse_args(argv)
+
+  if not all([options.inputs,
+              options.output,]):
+    l.LogError("Must supply --inputs and --output")
+    return 1
+
+  try:
+    pm = ProfileMerger(options.inputs.split(","), options.output,
+                       int(options.chunk_size), options.merge_program,
+                       options.multipliers)
+    pm.DoMerge()
+    retval = 0
+  except:
+    retval = 1
+  finally:
+    print "My work is done..."
+  return retval
+
+
+if __name__ == "__main__":
+  retval = Main(sys.argv)
+  sys.exit(retval)
diff --git a/fdo_scripts/divide_and_merge_profiles_test.py b/fdo_scripts/divide_and_merge_profiles_test.py
new file mode 100755
index 00000000..f42db4e9
--- /dev/null
+++ b/fdo_scripts/divide_and_merge_profiles_test.py
@@ -0,0 +1,132 @@
+#!/usr/bin/python2.6
+#
+# Copyright 2010 Google Inc. All Rights Reserved.
+
+__author__ = "asharif@google.com (Ahmad Sharif)"
+
+import os
+import random
+import shutil
+import tempfile
+import unittest
+
+from utils import command_executer
+from utils import misc
+
+
+class DivideAndMergeProfilesTest(unittest.TestCase):
+  def tearDown(self):
+    shutil.rmtree(self._program_dir)
+    for profile_dir in self._profile_dirs:
+      shutil.rmtree(profile_dir)
+
+  def setUp(self):
+    self._ce = command_executer.GetCommandExecuter()
+    self._program_dir = tempfile.mkdtemp()
+    self._writeProgram()
+    self._writeMakefile()
+    with misc.WorkingDirectory(self._program_dir):
+      self._ce.RunCommand("make")
+    num_profile_dirs = 2
+    self._profile_dirs = []
+    for i in range(num_profile_dirs):
+      profile_dir = tempfile.mkdtemp()
+      command = ("GCOV_PREFIX_STRIP=%s GCOV_PREFIX=$(/bin/pwd) "
+                 " %s/program" %
+                 (profile_dir.count("/"),
+                  self._program_dir))
+      with misc.WorkingDirectory(profile_dir):
+        self._ce.RunCommand(command)
+      self._profile_dirs.append(profile_dir)
+    self._merge_program = "/home/build/static/projects/crosstool/profile-merge/v14.5/profile_merge.par"
+
+  def _writeMakefile(self):
+    makefile_contents = """
+CC = gcc
+
+CFLAGS = -fprofile-generate
+
+SRCS=$(wildcard *.c)
+
+OBJS=$(SRCS:.c=.o)
+
+all: program
+
+program: $(OBJS)
+	$(CC) -o $@ $^ $(CFLAGS)
+
+%.o: %.c
+	$(CC) -c -o $@ $^ $(CFLAGS)"""
+    makefile = os.path.join(self._program_dir, "Makefile")
+    with open(makefile, "w") as f:
+      print >> f, makefile_contents
+
+  def _writeProgram(self, num_files=100):
+    for i in range(num_files):
+      current_file = os.path.join(self._program_dir, "%s.c" % i)
+      with open(current_file, "w") as f:
+        if i != num_files - 1:
+          print >> f, "extern void foo%s();" % (i + 1)
+          print >> f, "void foo%s(){foo%s();}" % (i, i + 1)
+        else:
+          print >> f, "void foo%s(){printf(\"\");}" % i
+        if i == 0:
+          print >> f, "int main(){foo%s(); return 0;}" % i
+
+  def testMerge(self):
+    reference_output = self._getReferenceOutput()
+    my_output = self._getMyOutput()
+
+    ret = self._diffOutputs(reference_output, my_output)
+    shutil.rmtree(my_output)
+    shutil.rmtree(reference_output)
+    self.assertTrue(ret == 0)
+
+  def _diffOutputs(self, reference, mine):
+    command = "diff -uNr %s %s" % (reference, mine)
+    return self._ce.RunCommand(command)
+
+  def _getMyOutput(self, args=""):
+    my_output = tempfile.mkdtemp()
+    my_merge_program = os.path.join(os.path.dirname(__file__),
+                                    "divide_and_merge_profiles.py")
+    command = ("python %s --inputs=%s --output=%s "
+               "--chunk_size=10 "
+               "--merge_program=%s "
+               "%s" %
+               (my_merge_program,
+                ",".join(self._profile_dirs),
+                my_output,
+                self._merge_program,
+                args))
+    self._ce.RunCommand(command)
+    return my_output
+
+  def _getReferenceOutput(self, args=""):
+    # First do a regular merge.
+    reference_output = tempfile.mkdtemp()
+    command = ("%s --inputs=%s --output=%s %s" %
+               (self._merge_program,
+                ",".join(self._profile_dirs),
+                reference_output,
+                args))
+    self._ce.RunCommand(command)
+    return reference_output
+
+  def testMergeWithMultipliers(self):
+    num_profiles = len(self._profile_dirs)
+    multipliers = [str(random.randint(0, num_profiles)) \
+                   for _ in range(num_profiles)]
+    args = "--multipliers=%s" % ",".join(multipliers)
+
+    reference_output = self._getReferenceOutput(args)
+    my_output = self._getMyOutput(args)
+
+    ret = self._diffOutputs(reference_output, my_output)
+
+    shutil.rmtree(my_output)
+    shutil.rmtree(reference_output)
+    self.assertTrue(ret == 0)
+
+if __name__ == "__main__":
+  unittest.main()
diff --git a/fdo_scripts/profile_cycler.py b/fdo_scripts/profile_cycler.py
new file mode 100755
index 00000000..efdafffa
--- /dev/null
+++ b/fdo_scripts/profile_cycler.py
@@ -0,0 +1,205 @@
+#!/usr/bin/python2.6
+#
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Script to profile a page cycler, and get it back to the host."""
+
+import copy
+import optparse
+import os
+import pickle
+import re
+import sys
+import tempfile
+import time
+
+import build_chrome_browser
+import cros_login
+import lock_machine
+import run_tests
+from utils import command_executer
+from utils import logger
+from utils import misc
+
+
+class CyclerProfiler:
+  REMOTE_TMP_DIR = "/tmp"
+
+  def __init__(self, chromeos_root, board, cycler, profile_dir, remote):
+    self._chromeos_root = chromeos_root
+    self._cycler = cycler
+    self._profile_dir = profile_dir
+    self._remote = remote
+    self._board = board
+    self._ce = command_executer.GetCommandExecuter()
+    self._l = logger.GetLogger()
+
+    self._gcov_prefix = os.path.join(self.REMOTE_TMP_DIR,
+                                     self._GetProfileDir())
+
+  def _GetProfileDir(self):
+    return misc.GetCtargetFromBoard(self._board, self._chromeos_root)
+
+  def _CopyTestData(self):
+    page_cycler_dir = os.path.join(self._chromeos_root,
+                                   "distfiles",
+                                   "target",
+                                   "chrome-src-internal",
+                                   "src",
+                                   "data",
+                                   "page_cycler")
+    if not os.path.isdir(page_cycler_dir):
+      raise Exception("Page cycler dir %s not found!" % page_cycler_dir)
+    self._ce.CopyFiles(page_cycler_dir,
+                       os.path.join(self.REMOTE_TMP_DIR, "page_cycler"),
+                       dest_machine=self._remote,
+                       chromeos_root=self._chromeos_root,
+                       recursive=True,
+                       dest_cros=True)
+
+  def _PrepareTestData(self):
+    # chmod files so everyone can read them.
+    command = ("cd %s && find page_cycler -type f | xargs chmod a+r" %
+               self.REMOTE_TMP_DIR)
+    self._ce.CrosRunCommand(command, chromeos_root=self._chromeos_root,
+                            machine=self._remote)
+    command = ("cd %s && find page_cycler -type d | xargs chmod a+rx" %
+               self.REMOTE_TMP_DIR)
+    self._ce.CrosRunCommand(command, chromeos_root=self._chromeos_root,
+                            machine=self._remote)
+
+  def _CopyProfileToHost(self):
+    dest_dir = os.path.join(self._profile_dir,
+                            os.path.basename(self._gcov_prefix))
+    # First remove the dir if it exists already
+    if os.path.exists(dest_dir):
+      command = "rm -rf %s" % dest_dir
+      self._ce.RunCommand(command)
+
+    # Strip out the initial prefix for the Chrome directory before doing the
+    # copy.
+    chrome_dir_prefix = misc.GetChromeSrcDir()
+
+    command = "mkdir -p %s" % dest_dir
+    self._ce.RunCommand(command)
+    self._ce.CopyFiles(self._gcov_prefix,
+                       dest_dir,
+                       src_machine=self._remote,
+                       chromeos_root=self._chromeos_root,
+                       recursive=True,
+                       src_cros=True)
+
+  def _RemoveRemoteProfileDir(self):
+    command = "rm -rf %s" % self._gcov_prefix
+    self._ce.CrosRunCommand(command, chromeos_root=self._chromeos_root,
+                            machine=self._remote)
+
+  def _LaunchCycler(self, cycler):
+    command = ("DISPLAY=:0 "
+               "XAUTHORITY=/home/chronos/.Xauthority "
+               "GCOV_PREFIX=%s "
+               "GCOV_PREFIX_STRIP=3 "
+               "/opt/google/chrome/chrome "
+               "--no-sandbox "
+               "--renderer-clean-exit "
+               "--user-data-dir=$(mktemp -d) "
+               "--url \"file:///%s/page_cycler/%s/start.html?iterations=10&auto=1\" "
+               "--enable-file-cookies "
+               "--no-first-run "
+               "--js-flags=expose_gc &" %
+               (self._gcov_prefix,
+                self.REMOTE_TMP_DIR,
+                cycler))
+
+    self._ce.CrosRunCommand(command, chromeos_root=self._chromeos_root,
+                            machine=self._remote,
+                            command_timeout=60)
+
+  def _PkillChrome(self, signal="9"):
+    command = "pkill -%s chrome" % signal
+    self._ce.CrosRunCommand(command, chromeos_root=self._chromeos_root,
+                            machine=self._remote)
+
+  def DoProfile(self):
+    # Copy the page cycler data to the remote
+    self._CopyTestData()
+    self._PrepareTestData()
+    self._RemoveRemoteProfileDir()
+
+    for cycler in self._cycler.split(","):
+      self._ProfileOneCycler(cycler)
+
+    # Copy the profile back
+    self._CopyProfileToHost()
+
+  def _ProfileOneCycler(self, cycler):
+    # With aura, all that's needed is a stop/start ui.
+    self._PkillChrome()
+    cros_login.RestartUI(self._remote, self._chromeos_root, login=False)
+    # Run the cycler
+    self._LaunchCycler(cycler)
+    self._PkillChrome(signal="INT")
+    # Let libgcov dump the profile.
+    # TODO(asharif): There is a race condition here. Fix it later.
+    time.sleep(30)
+
+
+def Main(argv):
+  """The main function."""
+  # Common initializations
+###  command_executer.InitCommandExecuter(True)
+  command_executer.InitCommandExecuter()
+  l = logger.GetLogger()
+  ce = command_executer.GetCommandExecuter()
+  parser = optparse.OptionParser()
+  parser.add_option("--cycler",
+                    dest="cycler",
+                    default="alexa_us",
+                    help=("Comma-separated cyclers to profile. "
+                          "Example: alexa_us,moz,moz2"
+                          "Use all to profile all cyclers."))
+  parser.add_option("--chromeos_root",
+                    dest="chromeos_root",
+                    default="../../",
+                    help="Output profile directory.")
+  parser.add_option("--board",
+                    dest="board",
+                    default="x86-zgb",
+                    help="The target board.")
+  parser.add_option("--remote",
+                    dest="remote",
+                    help=("The remote chromeos machine that"
+                          " has the profile image."))
+  parser.add_option("--profile_dir",
+                    dest="profile_dir",
+                    default="profile_dir",
+                    help="Store profiles in this directory.")
+
+  options, _ = parser.parse_args(argv)
+
+  all_cyclers = ["alexa_us", "bloat", "dhtml", "dom",
+                 "intl1", "intl2", "morejs", "morejsnp",
+                 "moz", "moz2"]
+
+  if options.cycler == "all":
+    options.cycler = ",".join(all_cyclers)
+
+  try:
+    cp = CyclerProfiler(options.chromeos_root,
+                        options.board,
+                        options.cycler,
+                        options.profile_dir,
+                        options.remote)
+    cp.DoProfile()
+    retval = 0
+  except Exception as e:
+    retval = 1
+    print e
+  finally:
+    print "Exiting..."
+  return retval
+
+
+if __name__ == "__main__":
+  retval = Main(sys.argv)
+  sys.exit(retval)
diff --git a/fdo_scripts/summarize_hot_blocks.py b/fdo_scripts/summarize_hot_blocks.py
new file mode 100644
index 00000000..c19f5477
--- /dev/null
+++ b/fdo_scripts/summarize_hot_blocks.py
@@ -0,0 +1,184 @@
+#!/usr/bin/python2.6
+#
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Summarize hottest basic blocks found while doing a ChromeOS FDO build.
+
+Here is an example execution:
+
+  summarize_hot_blocks.py
+   --data_dir=~/chromeos/chroot/var/cache/chromeos-chrome/ --cutoff=10000
+   --output_dir=/home/x/y
+
+With the cutoff, it will ignore any basic blocks that have a count less
+than what is specified (in this example 10000)
+The script looks inside the directory (this is typically a directory where
+the object files are generated) for files with *.profile and *.optimized
+suffixes. To get these, the following flags were added to the compiler
+invokation within vanilla_vs_fdo.py in the profile-use phase.
+
+              "-fdump-tree-optimized-blocks-lineno "
+              "-fdump-ipa-profile-blocks-lineno "
+
+Here is an example of the *.profile and *.optimized files contents:
+
+# BLOCK 7 freq:3901 count:60342, starting at line 92
+# PRED: 6 [39.0%]  count:60342 (true,exec)
+  [url_canon_internal.cc : 92:28] MEM[(const char * *)source_6(D) + 16B] = D.28080_17;
+  [url_canon_internal.cc : 93:41] MEM[(struct Component *)parsed_4(D) + 16B] = MEM[(const struct Component &)repl_1(D) + 80];
+# SUCC: 8 [100.0%]  count:60342 (fallthru,exec)
+# BLOCK 8 freq:10000 count:154667, starting at line 321
+# PRED: 7 [100.0%]  count:60342 (fallthru,exec) 6 [61.0%]  count:94325 (false,exec)
+  [url_canon_internal.cc : 321:51] # DEBUG D#10 => [googleurl/src/url_canon_internal.cc : 321] &parsed_4(D)->host
+
+this script finds the blocks with highest count and shows the first line
+of each block so that it is easy to identify the origin of the basic block.
+
+"""
+
+__author__ = "llozano@google.com (Luis Lozano)"
+
+import optparse
+import os
+import re
+import shutil
+import sys
+import tempfile
+
+from utils import command_executer
+
+
+# Given a line, check if it has a block count and return it.
+# Return -1 if there is no match
+def GetBlockCount(line):
+  match_obj = re.match(".*# BLOCK \d+ .*count:(\d+)", line)
+  if match_obj:
+    return int(match_obj.group(1))
+  else:
+    return -1
+
+
+class Collector(object):
+  def __init__(self, data_dir, cutoff, output_dir, tempdir):
+    self._data_dir = data_dir
+    self._cutoff = cutoff
+    self._output_dir = output_dir
+    self._tempdir = tempdir
+    self._ce = command_executer.GetCommandExecuter()
+
+  def CollectFileList(self, file_exp, list_file):
+    command = ("find %s -type f -name '%s' > %s" %
+               (self._data_dir, file_exp,
+                os.path.join(self._tempdir, list_file)))
+    ret = self._ce.RunCommand(command)
+    if ret:
+      raise Exception("Failed: %s" % command)
+
+  def SummarizeLines(self, data_file):
+    sum_lines = []
+    search_lno = False
+    for line in data_file:
+      count = GetBlockCount(line)
+      if count != -1:
+        if count >= self._cutoff:
+          search_lno = True
+          sum_line = line.strip()
+          sum_count = count
+      # look for a line that starts with line number information
+      elif search_lno and re.match("^\s*\[.*: \d*:\d*]", line):
+        search_lno = False
+        sum_lines.append("%d:%s: %s %s" %
+                         (sum_count, data_file.name, sum_line, line))
+    return sum_lines
+
+  # Look for blocks in the data file that have a count larger than the cutoff
+  # and generate a sorted summary file of the hottest blocks.
+  def SummarizeFile(self, data_file, sum_file):
+    with open(data_file, "r") as f:
+      sum_lines = self.SummarizeLines(f)
+
+    # sort reverse the list in place by the block count number
+    sum_lines.sort(key=GetBlockCount, reverse=True)
+
+    with open(sum_file, "w") as sf:
+      sf.write("".join(sum_lines))
+
+    print "Generated file Summary: ", sum_file
+
+  # Find hottest blocks in the list of files, generate a sorted summary for
+  # each file and then do a sorted merge of all the summaries.
+  def SummarizeList(self, list_file, summary_file):
+    with open(os.path.join(self._tempdir, list_file)) as f:
+      sort_list = []
+      for file_name in f:
+        file_name = file_name.strip()
+        sum_file = "%s.sum" % file_name
+        sort_list.append("%s%s" % (sum_file, chr(0)))
+        self.SummarizeFile(file_name, sum_file)
+
+    tmp_list_file = os.path.join(self._tempdir, "file_list.dat")
+    with open(tmp_list_file, "w") as file_list_file:
+      for x in sort_list:
+        file_list_file.write(x)
+
+    merge_command = ("sort -nr -t: -k1 --merge --files0-from=%s > %s " %
+                     (tmp_list_file, summary_file))
+
+    ret = self._ce.RunCommand(merge_command)
+    if ret:
+      raise Exception("Failed: %s" % merge_command)
+    print "Generated general summary: ", summary_file
+
+  def SummarizePreOptimized(self, summary_file):
+    self.CollectFileList("*.profile", "chrome.profile.list")
+    self.SummarizeList("chrome.profile.list",
+                       os.path.join(self._output_dir, summary_file))
+
+  def SummarizeOptimized(self, summary_file):
+    self.CollectFileList("*.optimized", "chrome.optimized.list")
+    self.SummarizeList("chrome.optimized.list",
+                       os.path.join(self._output_dir, summary_file))
+
+
+def Main(argv):
+  command_executer.InitCommandExecuter()
+  usage = ("usage: %prog --data_dir=<dir> --cutoff=<value> "
+           "--output_dir=<dir> [--keep_tmp]")
+  parser = optparse.OptionParser(usage=usage)
+  parser.add_option("--data_dir",
+                    dest="data_dir",
+                    help=("directory where the FDO (*.profile and "
+                          "*.optimized) files are located"))
+  parser.add_option("--cutoff",
+                    dest="cutoff",
+                    help="Minimum count to consider for each basic block")
+  parser.add_option("--output_dir",
+                    dest="output_dir",
+                    help=("directory where summary data will be generated"
+                          "(pre_optimized.txt, optimized.txt)"))
+  parser.add_option("--keep_tmp",
+                    action="store_true",
+                    dest="keep_tmp",
+                    default=False,
+                    help=("Keep directory with temporary files"
+                          "(for debugging purposes)"))
+  options = parser.parse_args(argv)[0]
+  if not all((options.data_dir, options.cutoff, options.output_dir)):
+    parser.print_help()
+    sys.exit(1)
+
+  tempdir = tempfile.mkdtemp()
+
+  co = Collector(options.data_dir, int(options.cutoff), options.output_dir,
+                 tempdir)
+  co.SummarizePreOptimized("pre_optimized.txt")
+  co.SummarizeOptimized("optimized.txt")
+
+  if not options.keep_tmp:
+    shutil.rmtree(tempdir, ignore_errors=True)
+
+  return 0
+
+if __name__ == "__main__":
+  retval = Main(sys.argv)
+  sys.exit(retval)
diff --git a/fdo_scripts/vanilla_vs_fdo.py b/fdo_scripts/vanilla_vs_fdo.py
new file mode 100644
index 00000000..3cf5f4b6
--- /dev/null
+++ b/fdo_scripts/vanilla_vs_fdo.py
@@ -0,0 +1,334 @@
+#!/usr/bin/python2.6
+#
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Script to build chrome with FDO and compare performance against no FDO."""
+
+import getpass
+import optparse
+import os
+import sys
+
+import image_chromeos
+import setup_chromeos
+from utils import command_executer
+from utils import misc
+from utils import logger
+
+
+class Patcher(object):
+  def __init__(self, dir_to_patch, patch_file):
+    self._dir_to_patch = dir_to_patch
+    self._patch_file = patch_file
+    self._base_patch_command = "patch -p0 %%s < %s" % patch_file
+    self._ce = command_executer.GetCommandExecuter()
+
+  def _RunPatchCommand(self, args):
+    patch_command = self._base_patch_command % args
+    command = ("cd %s && %s" % (self._dir_to_patch,
+                                patch_command))
+    return self._ce.RunCommand(command)
+
+  def _ApplyPatch(self, args):
+    full_args = "%s --dry-run" % args
+    ret = self._RunPatchCommand(full_args)
+    if ret:
+      raise Exception("Patch dry run failed!")
+    ret = self._RunPatchCommand(args)
+    if ret:
+      raise Exception("Patch application failed!")
+
+  def __enter__(self):
+    self._ApplyPatch("")
+
+  def __exit__(self, type, value, traceback):
+    self._ApplyPatch("-R")
+
+
+class FDOComparator(object):
+  def __init__(self, board, remotes, ebuild_version, plus_pgo, minus_pgo,
+               update_pgo, chromeos_root):
+    self._board = board
+    self._remotes = remotes
+    self._ebuild_version = ebuild_version
+    self._remote = remotes.split(",")[0]
+    self._chromeos_root = chromeos_root
+    self._profile_dir = "profile_dir"
+    self._profile_path = os.path.join(self._chromeos_root,
+                                      "src",
+                                      "scripts",
+                                      os.path.basename(self._profile_dir))
+    self._plus_pgo = plus_pgo
+    self._minus_pgo = minus_pgo
+    self._update_pgo = update_pgo
+
+    self._ce = command_executer.GetCommandExecuter()
+    self._l = logger.GetLogger()
+
+  def _CheckoutChromeOS(self):
+    if not os.path.exists(self._chromeos_root):
+      setup_chromeos_args = [setup_chromeos.__file__,
+                             "--dir=%s" % self._chromeos_root,
+                             "--minilayout"]
+      setup_chromeos.Main(setup_chromeos_args)
+
+  def _BuildChromeOSUsingBinaries(self):
+    image_dir = misc.GetImageDir(self._chromeos_root, self._board)
+    command = "equery-%s l chromeos" % self._board
+    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
+    if ret:
+      command = misc.GetSetupBoardCommand(self._board,
+                                           usepkg=True)
+      ret = self._ce.ChrootRunCommand(self._chromeos_root,
+                                      command)
+      if ret:
+        raise Exception("Couldn't run setup_board!")
+      command = misc.GetBuildPackagesCommand(self._board,
+                                              True)
+      ret = self._ce.ChrootRunCommand(self._chromeos_root,
+                                    command)
+      if ret:
+        raise Exception("Couldn't run build_packages!")
+
+  def _ReportMismatches(self, build_log):
+    mismatch_signature = "-Wcoverage-mismatch"
+    mismatches = build_log.count(mismatch_signature)
+    self._l.LogOutput("Total mismatches: %s" % mismatches)
+    stale_files = set([])
+    for line in build_log.splitlines():
+      if mismatch_signature in line:
+        filename = line.split(":")[0]
+        stale_files.add(filename)
+    self._l.LogOutput("Total stale files: %s" % len(stale_files))
+
+  def _BuildChromeAndImage(self, ebuild_version="", env_dict={}, cflags="",
+                           cxxflags="", ldflags="", label="",
+                           build_image_args=""):
+    env_string = misc.GetEnvStringFromDict(env_dict)
+    if not label:
+      label = " ".join([env_string,
+                        cflags,
+                        cxxflags,
+                        ldflags,
+                        ebuild_version])
+      label = label.strip()
+      label = misc.GetFilenameFromString(label)
+    if not misc.DoesLabelExist(self._chromeos_root, self._board, label):
+      build_chrome_browser_args = ["--clean",
+                                   "--chromeos_root=%s" % self._chromeos_root,
+                                   "--board=%s" % self._board,
+                                   "--env=%r" % env_string,
+                                   "--cflags=%r" % cflags,
+                                   "--cxxflags=%r" % cxxflags,
+                                   "--ldflags=%r" % ldflags,
+                                   "--ebuild_version=%s" % ebuild_version,
+                                   "--build_image_args=%s" % build_image_args]
+
+      build_chrome_browser = os.path.join(os.path.dirname(__file__),
+                                          "..",
+                                          "build_chrome_browser.py")
+      command = "python %s %s" % (build_chrome_browser,
+                                  " ".join(build_chrome_browser_args))
+      ret, out, err = self._ce.RunCommand(command,
+                                          return_output=True)
+      if "-fprofile-use" in cxxflags:
+        self._ReportMismatches(out)
+
+      if ret:
+        raise Exception("Couldn't build chrome browser!")
+      misc.LabelLatestImage(self._chromeos_root, self._board, label)
+    return label
+
+  def _TestLabels(self, labels):
+    experiment_file = "pgo_experiment.txt"
+    experiment_header = """
+    board: %s
+    remote: %s
+    """ % (self._board, self._remotes)
+    experiment_tests = """
+    benchmark: desktopui_PyAutoPerfTests {
+      iterations: 1
+    }
+    """
+    with open(experiment_file, "w") as f:
+      print >>f, experiment_header
+      print >>f, experiment_tests
+      for label in labels:
+        # TODO(asharif): Fix crosperf so it accepts labels with symbols
+        crosperf_label = label
+        crosperf_label = crosperf_label.replace("-", "minus")
+        crosperf_label = crosperf_label.replace("+", "plus")
+        experiment_image = """
+        %s {
+          chromeos_image: %s
+        }
+        """ % (crosperf_label,
+               os.path.join(misc.GetImageDir(self._chromeos_root, self._board),
+                            label,
+                            "chromiumos_test_image.bin"))
+        print >>f, experiment_image
+    crosperf = os.path.join(os.path.dirname(__file__),
+                            "..",
+                            "crosperf",
+                            "crosperf")
+    command = "%s %s" % (crosperf, experiment_file)
+    ret = self._ce.RunCommand(command)
+    if ret:
+      raise Exception("Couldn't run crosperf!")
+
+  def _ImageRemote(self, label):
+    image_path = os.path.join(misc.GetImageDir(self._chromeos_root,
+                                                self._board),
+                              label,
+                              "chromiumos_test_image.bin")
+    image_chromeos_args = [image_chromeos.__file__,
+                           "--chromeos_root=%s" % self._chromeos_root,
+                           "--image=%s" % image_path,
+                           "--remote=%s" % self._remote,
+                           "--board=%s" % self._board]
+    image_chromeos.Main(image_chromeos_args)
+
+  def _ProfileRemote(self):
+    profile_cycler = os.path.join(os.path.dirname(__file__),
+                                  "profile_cycler.py")
+    profile_cycler_args = ["--chromeos_root=%s" % self._chromeos_root,
+                           "--cycler=all",
+                           "--board=%s" % self._board,
+                           "--profile_dir=%s" % self._profile_path,
+                           "--remote=%s" % self._remote]
+    command = "python %s %s" % (profile_cycler, " ".join(profile_cycler_args))
+    ret = self._ce.RunCommand(command)
+    if ret:
+      raise Exception("Couldn't profile cycler!")
+
+  def _BuildGenerateImage(self):
+    # TODO(asharif): add cflags as well.
+    labels_list = ["fprofile-generate", self._ebuild_version]
+    label = "_".join(labels_list)
+    generate_label = self._BuildChromeAndImage(
+        env_dict={"USE": "chrome_internal -pgo pgo_generate"},
+        label=label,
+        ebuild_version=self._ebuild_version,
+        build_image_args="--rootfs_boost_size=400")
+    return generate_label
+
+  def _BuildUseImage(self):
+    ctarget = misc.GetCtargetFromBoard(self._board, self._chromeos_root)
+    chroot_profile_dir = os.path.join("/home/%s/trunk" % getpass.getuser(),
+                                      "src",
+                                      "scripts",
+                                      self._profile_dir,
+                                      ctarget)
+    cflags = ("-fprofile-use "
+              "-fprofile-correction "
+              "-Wno-error "
+              "-fdump-tree-optimized-blocks-lineno "
+              "-fdump-ipa-profile-blocks-lineno "
+              "-fno-vpt "
+              "-fprofile-dir=%s" %
+              chroot_profile_dir)
+    labels_list = ["updated_pgo", self._ebuild_version]
+    label = "_".join(labels_list)
+    pgo_use_label = self._BuildChromeAndImage(
+        env_dict={"USE": "chrome_internal -pgo"},
+        cflags=cflags,
+        cxxflags=cflags,
+        ldflags=cflags,
+        label=label,
+        ebuild_version=self._ebuild_version)
+    return pgo_use_label
+
+  def DoAll(self):
+    self._CheckoutChromeOS()
+    self._BuildChromeOSUsingBinaries()
+    labels = []
+
+    if self._minus_pgo:
+      minus_pgo = self._BuildChromeAndImage(env_dict={"USE": "chrome_internal -pgo"},
+                                            ebuild_version=self._ebuild_version)
+      labels.append(minus_pgo)
+    if self._plus_pgo:
+      plus_pgo = self._BuildChromeAndImage(env_dict={"USE": "chrome_internal pgo"},
+                                           ebuild_version=self._ebuild_version)
+      labels.append(plus_pgo)
+
+    if self._update_pgo:
+      if not os.path.exists(self._profile_path):
+        # Build Chrome with -fprofile-generate
+        generate_label = self._BuildGenerateImage()
+        # Image to the remote box.
+        self._ImageRemote(generate_label)
+        # Profile it using all page cyclers.
+        self._ProfileRemote()
+
+      # Use the profile directory to rebuild it.
+      updated_pgo_label = self._BuildUseImage()
+      labels.append(updated_pgo_label)
+
+    # Run crosperf on all images now.
+    self._TestLabels(labels)
+    return 0
+
+
+def Main(argv):
+  """The main function."""
+  # Common initializations
+###  command_executer.InitCommandExecuter(True)
+  command_executer.InitCommandExecuter()
+  parser = optparse.OptionParser()
+  parser.add_option("--remote",
+                    dest="remote",
+                    help="Remote machines to run tests on.")
+  parser.add_option("--board",
+                    dest="board",
+                    default="x86-zgb",
+                    help="The target board.")
+  parser.add_option("--ebuild_version",
+                    dest="ebuild_version",
+                    default="",
+                    help="The Chrome ebuild version to use.")
+  parser.add_option("--plus_pgo",
+                    dest="plus_pgo",
+                    action="store_true",
+                    default=False,
+                    help="Build USE=+pgo.")
+  parser.add_option("--minus_pgo",
+                    dest="minus_pgo",
+                    action="store_true",
+                    default=False,
+                    help="Build USE=-pgo.")
+  parser.add_option("--update_pgo",
+                    dest="update_pgo",
+                    action="store_true",
+                    default=False,
+                    help="Update pgo and build Chrome with the update.")
+  parser.add_option("--chromeos_root",
+                    dest="chromeos_root",
+                    default=False,
+                    help="The chromeos root directory")
+  options, _ = parser.parse_args(argv)
+  if not options.board:
+    print "Please give a board."
+    return 1
+  if not options.remote:
+    print "Please give at least one remote machine."
+    return 1
+  if not options.chromeos_root:
+    print "Please provide the chromeos root directory."
+    return 1
+  if not any((options.minus_pgo, options.plus_pgo, options.update_pgo)):
+    print "Please provide at least one build option."
+    return 1
+  fc = FDOComparator(options.board,
+                     options.remote,
+                     options.ebuild_version,
+                     options.plus_pgo,
+                     options.minus_pgo,
+                     options.update_pgo,
+                     os.path.expanduser(options.chromeos_root))
+  return fc.DoAll()
+
+
+if __name__ == "__main__":
+  retval = Main(sys.argv)
+  sys.exit(retval)