Cleaned up directory after copy of tools from perforce directory

Got rid of stale copies of some tools like "crosperf" and moved all files under v14 directory (that came from perforce) into the top directory. BUG=None TEST=None Change-Id: I408d17a36ceb00e74db71403d2351fd466a14f8e Reviewed-on: https://gerrit-int.chromium.org/33887 Tested-by: Luis Lozano <llozano@chromium.org> Reviewed-by: Yunlian Jiang <yunlian@google.com> Commit-Queue: Luis Lozano <llozano@chromium.org>
author: Luis Lozano <llozano@chromium.org> 2013-03-15 14:44:13 -0700
committer: ChromeBot <chrome-bot@google.com> 2013-03-15 15:51:37 -0700
commit: f81680c018729fd4499e1e200d04b48c4b90127c (patch)
tree: 940608da8374604b82edfdb2d7df55d065f05d4c /cwp
parent: 2296ee0b914aba5bba07becab4ff68884ce9b8a5 (diff)
download: toolchain-utils-f81680c018729fd4499e1e200d04b48c4b90127c.tar.gz
9 files changed, 824 insertions, 0 deletions
diff --git a/cwp/bartlett/app.yaml b/cwp/bartlett/app.yaml
new file mode 100644
index 00000000..60010f70
--- /dev/null
+++ b/cwp/bartlett/app.yaml
@@ -0,0 +1,22 @@
+application: chromeoswideprofiling
+version: 1
+runtime: python
+api_version: 1
+
+handlers:
+- url: /favicon.ico
+  static_files: static/favicon.ico
+  upload: static/favicon.ico
+- url: /remote_api
+  script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
+  login: admin
+- url: /
+  script: server.py
+- url: /upload
+  script: server.py
+- url: /serve
+  script: server.py
+- url: /serve/.*
+  script: server.py
+- url: /del/.*
+  script: server.py
diff --git a/cwp/bartlett/server.py b/cwp/bartlett/server.py
new file mode 100755
index 00000000..3e4f1be7
--- /dev/null
+++ b/cwp/bartlett/server.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python2.6
+# Copyright 2012 Google Inc. All Rights Reserved.
+# Author: mrdmnd@ (Matt Redmond)
+# Based off of code in //depot/google3/experimental/mobile_gwp
+"""Code to transport profile data between a user's machine and the CWP servers.
+    Pages:
+    "/": the main page for the app, left blank so that users cannot access
+         the file upload but left in the code for debugging purposes
+    "/upload": Updates the datastore with a new file. the upload depends on
+               the format which is templated on the main page ("/")
+               input includes:
+                    profile_data: the zipped file containing profile data
+                    board:  the architecture we ran on
+                    chromeos_version: the chromeos_version
+    "/serve": Lists all of the files in the datastore. Each line is a new entry
+              in the datastore. The format is key~date, where key is the entry's
+              key in the datastore and date is the file upload time and date.
+              (Authentication Required)
+    "/serve/([^/]+)?": For downloading a file of profile data, ([^/]+)? means
+                       any character sequence so to download the file go to
+                       '/serve/$key' where $key is the datastore key of the file
+                       you want to download.
+                       (Authentication Required)
+    "/del/([^/]+)?": For deleting an entry in the datastore. To use go to
+                     '/del/$key' where $key is the datastore key of the entry
+                     you want to be deleted form the datastore.
+                     (Authentication Required)
+    TODO: Add more extensive logging"""
+
+import cgi
+import logging
+import md5
+import urllib
+
+from google.appengine.api import users
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+
+logging.getLogger().setLevel(logging.DEBUG)
+
+
+class FileEntry(db.Model):
+  profile_data = db.BlobProperty()                # The profile data
+  date = db.DateTimeProperty(auto_now_add=True)   # Date it was uploaded
+  data_md5 = db.ByteStringProperty()              # md5 of the profile data
+  board = db.StringProperty()                     # board arch
+  chromeos_version = db.StringProperty()          # ChromeOS version
+
+
+class MainPage(webapp.RequestHandler):
+  """Main page only used as the form template, not actually displayed."""
+
+  def get(self, response=""):  # pylint: disable-msg=C6409
+    if response:
+      self.response.out.write("<html><body>")
+      self.response.out.write("""<br>
+        <form action="/upload" enctype="multipart/form-data" method="post">
+          <div><label>Profile Data:</label></div>
+          <div><input type="file" name="profile_data"/></div>
+          <div><label>Board</label></div>
+          <div><input type="text" name="board"/></div>
+          <div><label>ChromeOS Version</label></div>
+          <div><input type="text" name="chromeos_version"></div>
+          <div><input type="submit" value="send" name="submit"></div>
+        </form>
+      </body>
+      </html>""")
+
+
+class Upload(webapp.RequestHandler):
+  """Handler for uploading data to the datastore, accessible by anyone."""
+
+  def post(self):  # pylint: disable-msg=C6409
+    """Takes input based on the main page's form."""
+    getfile = FileEntry()
+    f1 = self.request.get("profile_data")
+    getfile.profile_data = db.Blob(f1)
+    getfile.data_md5 = md5.new(f1).hexdigest()
+    getfile.board = self.request.get("board")
+    getfile.chromeos_version = self.request.get("chromeos_version")
+    getfile.put()
+    self.response.out.write(getfile.key())
+    #self.redirect('/')
+
+
+class ServeHandler(webapp.RequestHandler):
+  """Given the entry's key in the database, output the profile data file. Only
+      accessible from @google.com accounts."""
+
+  def get(self, resource):  # pylint: disable-msg=C6409
+    if Authenticate(self):
+      file_key = str(urllib.unquote(resource))
+      request = db.get(file_key)
+      self.response.out.write(request.profile_data)
+
+
+class ListAll(webapp.RequestHandler):
+  """Displays all files uploaded. Only accessible by @google.com accounts."""
+
+  def get(self):  # pylint: disable-msg=C6409
+    """Displays all information in FileEntry, ~ delimited."""
+    if Authenticate(self):
+      query_str = "SELECT * FROM FileEntry ORDER BY date ASC"
+      query = db.GqlQuery(query_str)
+      delimiter = "~"
+
+      for item in query:
+        display_list = [item.key(), item.date, item.data_md5,
+                        item.board, item.chromeos_version]
+        str_list = [cgi.escape(str(i)) for i in display_list]
+        self.response.out.write(delimiter.join(str_list)+"</br>")
+
+
+class DelEntries(webapp.RequestHandler):
+  """Deletes entries. Only accessible from @google.com accounts."""
+
+  def get(self, resource):  # pylint: disable-msg=C6409
+    """A specific entry is deleted, when the key is given."""
+    if Authenticate(self):
+      fkey = str(urllib.unquote(resource))
+      request = db.get(fkey)
+      if request:
+        db.delete(fkey)
+
+
+def Authenticate(webpage):
+  """Some urls are only accessible if logged in with a @google.com account."""
+  user = users.get_current_user()
+  if user is None:
+    webpage.redirect(users.create_login_url(webpage.request.uri))
+  elif user.email().endswith("@google.com"):
+    return True
+  else:
+    webpage.response.out.write("Not Authenticated")
+    return False
+
+
+def main():
+  application = webapp.WSGIApplication([
+      ("/", MainPage),
+      ("/upload", Upload),
+      ("/serve/([^/]+)?", ServeHandler),
+      ("/serve", ListAll),
+      ("/del/([^/]+)?", DelEntries),
+  ], debug=False)
+  run_wsgi_app(application)
+
+
+if __name__ == "__main__":
+  main()
+
diff --git a/cwp/bartlett/static/favicon.ico b/cwp/bartlett/static/favicon.ico
new file mode 100644
index 00000000..19b58c2e
--- /dev/null
+++ b/cwp/bartlett/static/favicon.ico
diff --git a/cwp/bartlett/test/server_tester.py b/cwp/bartlett/test/server_tester.py
new file mode 100644
index 00000000..e5a2341d
--- /dev/null
+++ b/cwp/bartlett/test/server_tester.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python2.6
+# Copyright 2012 Google Inc. All Rights Reserved.
+# Author: mrdmnd@ (Matt Redmond)
+"""A unit test for sending data to Bartlett. Requires poster module."""
+
+import cookielib
+import os
+import signal
+import subprocess
+import tempfile
+import time
+import unittest
+import urllib2
+
+from poster.encode import multipart_encode
+from poster.streaminghttp import register_openers
+
+
+SERVER_DIR = "../."
+SERVER_URL = "http://localhost:8080/"
+GET = "_ah/login?email=googler@google.com&action=Login&continue=%s"
+AUTH_URL = SERVER_URL + GET
+
+
+class ServerTest(unittest.TestCase):
+  """A unit test for the bartlett server. Tests upload, serve, and delete."""
+
+  def setUp(self):
+    """Instantiate the files and server needed to test upload functionality."""
+    self._server_proc = LaunchLocalServer()
+    self._jar = cookielib.LWPCookieJar()
+    self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._jar))
+
+    # We need these files to not delete when closed, because we have to reopen
+    # them in read mode after we write and close them.
+    self.profile_data = tempfile.NamedTemporaryFile(delete=False)
+
+    size = 16 * 1024
+    self.profile_data.write(os.urandom(size))
+
+  def tearDown(self):
+    self.profile_data.close()
+    os.remove(self.profile_data.name)
+    os.kill(self._server_proc.pid, signal.SIGINT)
+
+  def testIntegration(self):  # pylint: disable-msg=C6409
+    key = self._testUpload()
+    self._testListAll()
+    self._testServeKey(key)
+    self._testDelKey(key)
+
+  def _testUpload(self):  # pylint: disable-msg=C6409
+    register_openers()
+    data = {"profile_data": self.profile_data,
+            "board": "x86-zgb",
+            "chromeos_version": "2409.0.2012_06_08_1114"}
+    datagen, headers = multipart_encode(data)
+    request = urllib2.Request(SERVER_URL + "upload", datagen, headers)
+    response = urllib2.urlopen(request).read()
+    self.assertTrue(response)
+    return response
+
+  def _testListAll(self):  # pylint: disable-msg=C6409
+    request = urllib2.Request(AUTH_URL % (SERVER_URL + "serve"))
+    response = self.opener.open(request).read()
+    self.assertTrue(response)
+
+  def _testServeKey(self, key):  # pylint: disable-msg=C6409
+    request = urllib2.Request(AUTH_URL % (SERVER_URL + "serve/" + key))
+    response = self.opener.open(request).read()
+    self.assertTrue(response)
+
+  def _testDelKey(self, key):  # pylint: disable-msg=C6409
+    # There is no response to a delete request.
+    # We will check the listAll page to ensure there is no data.
+    request = urllib2.Request(AUTH_URL % (SERVER_URL + "del/" + key))
+    response = self.opener.open(request).read()
+    request = urllib2.Request(AUTH_URL % (SERVER_URL + "serve"))
+    response = self.opener.open(request).read()
+    self.assertFalse(response)
+
+
+def LaunchLocalServer():
+  """Launch and store an authentication cookie with a local server."""
+  proc = subprocess.Popen(["dev_appserver.py", "--clear_datastore", SERVER_DIR],
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  # Wait for server to come up
+  while True:
+    time.sleep(1)
+    try:
+      request = urllib2.Request(SERVER_URL + "serve")
+      response = urllib2.urlopen(request).read()
+      if response:
+        break
+    except urllib2.URLError:
+      continue
+  return proc
+
+
+if __name__ == "__main__":
+  unittest.main()
+
diff --git a/cwp/bartlett/update_appengine_server b/cwp/bartlett/update_appengine_server
new file mode 100755
index 00000000..f3812057
--- /dev/null
+++ b/cwp/bartlett/update_appengine_server
@@ -0,0 +1 @@
+appcfg.py --oauth2 update .
diff --git a/cwp/demo_pipeline.sh b/cwp/demo_pipeline.sh
new file mode 100644
index 00000000..d45c5c44
--- /dev/null
+++ b/cwp/demo_pipeline.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# These should be on the local filesystem. We'll be hitting it hard.
+DATA_DIR=/usr/local/google/home/${USER}/
+SYMBOL_CACHE=${DATA_DIR}cache/
+REPORT_DIR=${DATA_DIR}reports/
+SAMPLE_DIR=${DATA_DIR}samples/
+RECORD_FILE=/tmp/profiles.rio
+COLUMN_FILE=/tmp/profiles.cio
+mkdir -p ${SYMBOL_CACHE}
+mkdir -p ${REPORT_DIR}
+mkdir -p ${SAMPLE_DIR}
+
+# Directory that has the scripts app_engine_pull.py and symbolizer.py
+INTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/
+V14_DIR=$(dirname $(dirname ${INTERPRETER_DIR}))
+
+PYTHONPATH=$PYTHONPATH:$V14_DIR
+
+# Profile util binary
+PROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util
+
+# mr-convert binary
+MR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert
+
+CNS_LOC=/cns/ag-d/home/${USER}/profiledb/
+
+# Protofile location
+PROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto
+
+echo "0. Cleaning up old data."
+rm /tmp/profiles.*
+rm ${REPORT_DIR}*
+rm ${SAMPLE_DIR}*
+
+
+echo "Starting CWP Pipeline..."
+echo "1. Pulling samples to local filesystem from server."
+echo "   For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm"
+python ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR}
+echo "2. Symbolizing samples to perf reports. Hold on..."
+
+python ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE}
+echo "3. Loading reports into RecordIO format..."
+# Will need to make append_dir more clever / incremental
+${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR}
+echo "Done."
+echo "4. Converting records to columnio."
+${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC}
+echo "5. Uploading columnio to colossus."
+fileutil cp -f /tmp/profiles.cio-* ${CNS_LOC}
+echo "6. Let's try some dremel queries..."
+echo "   dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*"
+echo "   Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;"
+
diff --git a/cwp/interpreter/app_engine_pull.py b/cwp/interpreter/app_engine_pull.py
new file mode 100644
index 00000000..65f67940
--- /dev/null
+++ b/cwp/interpreter/app_engine_pull.py
@@ -0,0 +1,245 @@
+#!/usr/bin/python
+# Copyright 2012 Google Inc. All Rights Reserved.
+# Author: mrdmnd@ (Matt Redmond)
+"""A client to pull data from Bartlett.
+
+Inspired by //depot/google3/experimental/mobile_gwp/database/app_engine_pull.py
+
+The server houses perf.data.gz, board, chrome version for each upload.
+This script first authenticates with a proper @google.com account, then
+downloads a sample (if it's not already cached) and unzips perf.data
+
+  Authenticate(): Gets login info and returns an auth token
+  DownloadSamples(): Download and unzip samples.
+  _GetServePage(): Pulls /serve page from the app engine server
+  _DownloadSampleFromServer(): Downloads a local compressed copy of a sample
+  _UncompressSample(): Decompresses a sample, deleting the compressed version.
+"""
+import cookielib
+import getpass
+import gzip
+import optparse
+import os
+import urllib
+import urllib2
+
+SERVER_NAME = "http://chromeoswideprofiling.appspot.com"
+APP_NAME = "chromeoswideprofiling"
+DELIMITER = "~"
+
+
+def Authenticate(server_name):
+  """Gets credentials from user and attempts to retrieve auth token.
+     TODO: Accept OAuth2 instead of password.
+  Args:
+    server_name: (string) URL that the app engine code is living on.
+  Returns:
+    authtoken: (string) The authorization token that can be used
+                        to grab other pages.
+  """
+
+  if server_name.endswith("/"):
+    server_name = server_name.rstrip("/")
+  # Grab username and password from user through stdin.
+  username = raw_input("Email (must be @google.com account): ")
+  password = getpass.getpass("Password: ")
+  # Use a cookie to authenticate with GAE.
+  cookiejar = cookielib.LWPCookieJar()
+  opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
+  urllib2.install_opener(opener)
+  # Get an AuthToken from Google accounts service.
+  auth_uri = "https://www.google.com/accounts/ClientLogin"
+  authreq_data = urllib.urlencode({"Email": username,
+                                   "Passwd": password,
+                                   "service": "ah",
+                                   "source": APP_NAME,
+                                   "accountType": "HOSTED_OR_GOOGLE"})
+  auth_req = urllib2.Request(auth_uri, data=authreq_data)
+  try:
+    auth_resp = urllib2.urlopen(auth_req)
+  except urllib2.URLError:
+    print "Error logging in to Google accounts service."
+    return None
+  body = auth_resp.read()
+  # Auth response contains several fields.
+  # We care about the part after Auth=
+  auth_resp_dict = dict(x.split("=") for x in body.split("\n") if x)
+  authtoken = auth_resp_dict["Auth"]
+  return authtoken
+
+
+def DownloadSamples(server_name, authtoken, output_dir, start, stop):
+  """Download every sample and write unzipped version
+     to output directory.
+  Args:
+    server_name: (string) URL that the app engine code is living on.
+    authtoken:   (string) Authorization token.
+    output_dir   (string) Filepath to write output to.
+    start:       (int)    Index to start downloading from, starting at top.
+    stop:        (int)    Index to stop downloading, non-inclusive. -1 for end.
+  Returns:
+    None
+  """
+
+  if server_name.endswith("/"):
+    server_name = server_name.rstrip("/")
+
+  serve_page_string = _GetServePage(server_name, authtoken)
+  if serve_page_string is None:
+    print "Error getting /serve page."
+    return
+
+  sample_list = serve_page_string.split("</br>")
+  print "Will download:"
+  sample_list_subset = sample_list[start:stop]
+  for sample in sample_list_subset:
+    print sample
+  for sample in sample_list_subset:
+    assert sample, "Sample should be valid."
+    sample_info = [s.strip() for s in sample.split(DELIMITER)]
+    key = sample_info[0]
+    time = sample_info[1]
+    time = time.replace(" ", "_") # No space between date and time.
+    # sample_md5 = sample_info[2]
+    board = sample_info[3]
+    version = sample_info[4]
+
+    # Put a compressed copy of the samples in output directory.
+    _DownloadSampleFromServer(server_name, authtoken, key, time, board,
+                              version, output_dir)
+    _UncompressSample(key, time, board, version, output_dir)
+
+
+def _BuildFilenameFromParams(key, time, board, version):
+  """Return the filename for our sample.
+  Args:
+    key:  (string) Key indexing our sample in the datastore.
+    time: (string) Date that the sample was uploaded.
+    board: (string) Board that the sample was taken on.
+    version: (string) Version string from /etc/lsb-release
+  Returns:
+    filename (string)
+  """
+  filename = DELIMITER.join([key, time, board, version])
+  return filename
+
+
+def _DownloadSampleFromServer(server_name, authtoken, key, time, board,
+                              version, output_dir):
+  """Downloads sample_$(samplekey).gz to current dir.
+  Args:
+    server_name: (string) URL that the app engine code is living on.
+    authtoken:   (string) Authorization token.
+    key:  (string) Key indexing our sample in the datastore
+    time: (string) Date that the sample was uploaded.
+    board: (string) Board that the sample was taken on.
+    version: (string) Version string from /etc/lsb-release
+    output_dir:  (string) Filepath to write to output to.
+  Returns:
+    None
+  """
+  filename = _BuildFilenameFromParams(key, time, board, version)
+  compressed_filename = filename+".gz"
+
+  if os.path.exists(os.path.join(output_dir, filename)):
+    print "Already downloaded %s, skipping." % filename
+    return
+
+  serv_uri = server_name + "/serve/" + key
+  serv_args = {"continue": serv_uri, "auth": authtoken}
+  full_serv_uri = server_name + "/_ah/login?%s" % urllib.urlencode(serv_args)
+  serv_req = urllib2.Request(full_serv_uri)
+  serv_resp = urllib2.urlopen(serv_req)
+  f = open(os.path.join(output_dir, compressed_filename), "w+")
+  f.write(serv_resp.read())
+  f.close()
+
+
+def _UncompressSample(key, time, board, version, output_dir):
+  """Uncompresses a given sample.gz file and deletes the compressed version.
+  Args:
+    key: (string) Sample key to uncompress.
+    time: (string) Date that the sample was uploaded.
+    board: (string) Board that the sample was taken on.
+    version: (string) Version string from /etc/lsb-release
+    output_dir: (string) Filepath to find sample key in.
+  Returns:
+    None
+  """
+  filename = _BuildFilenameFromParams(key, time, board, version)
+  compressed_filename = filename+".gz"
+
+  if os.path.exists(os.path.join(output_dir, filename)):
+    print "Already decompressed %s, skipping." % filename
+    return
+
+  out_file = open(os.path.join(output_dir, filename), "wb")
+  in_file = gzip.open(os.path.join(output_dir, compressed_filename), "rb")
+  out_file.write(in_file.read())
+  in_file.close()
+  out_file.close()
+  os.remove(os.path.join(output_dir, compressed_filename))
+
+
+def _DeleteSampleFromServer(server_name, authtoken, key):
+  """Opens the /delete page with the specified key
+     to delete the sample off the datastore.
+    Args:
+      server_name: (string) URL that the app engine code is living on.
+      authtoken:   (string) Authorization token.
+      key:  (string) Key to delete.
+    Returns:
+      None
+  """
+
+  serv_uri = server_name + "/del/" + key
+  serv_args = {"continue": serv_uri, "auth": authtoken}
+  full_serv_uri = server_name + "/_ah/login?%s" % urllib.urlencode(serv_args)
+  serv_req = urllib2.Request(full_serv_uri)
+  urllib2.urlopen(serv_req)
+
+
+def _GetServePage(server_name, authtoken):
+  """Opens the /serve page and lists all keys.
+  Args:
+    server_name: (string) URL the app engine code is living on.
+    authtoken:   (string) Authorization token.
+  Returns:
+    The text of the /serve page (including HTML tags)
+  """
+
+  serv_uri = server_name + "/serve"
+  serv_args = {"continue": serv_uri, "auth": authtoken}
+  full_serv_uri = server_name + "/_ah/login?%s" % urllib.urlencode(serv_args)
+  serv_req = urllib2.Request(full_serv_uri)
+  serv_resp = urllib2.urlopen(serv_req)
+  return serv_resp.read()
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option("--output_dir", dest="output_dir", action="store",
+                    help="Path to output perf data files.")
+  parser.add_option("--start", dest="start_ind", action="store",
+                    default=0, help="Start index.")
+  parser.add_option("--stop", dest="stop_ind", action="store",
+                    default=-1, help="Stop index.")
+  options = parser.parse_args()[0]
+  if not options.output_dir:
+    print "Must specify --output_dir."
+    return 1
+  if not os.path.exists(options.output_dir):
+    print "Specified output_dir does not exist."
+    return 1
+
+  authtoken = Authenticate(SERVER_NAME)
+  if not authtoken:
+    print "Could not obtain authtoken, exiting."
+    return 1
+  DownloadSamples(SERVER_NAME, authtoken, options.output_dir,
+                  options.start_ind, options.stop_ind)
+  print "Downloaded samples."
+  return 0
+
+if __name__ == "__main__":
+  exit(main())
diff --git a/cwp/interpreter/symbolizer.py b/cwp/interpreter/symbolizer.py
new file mode 100644
index 00000000..3e589538
--- /dev/null
+++ b/cwp/interpreter/symbolizer.py
@@ -0,0 +1,131 @@
+#!/usr/bin/python
+# Copyright 2012 Google Inc. All Rights Reserved.
+"""A script that symbolizes perf.data files."""
+import optparse
+import os
+import shutil
+from subprocess import call
+from subprocess import PIPE
+from subprocess import Popen
+from utils import misc
+
+
+GSUTIL_CMD = "gsutil cp gs://chromeos-image-archive/%s-release/%s/debug.tgz %s"
+TAR_CMD = "tar -zxvf %s -C %s"
+PERF_BINARY = "/google/data/ro/projects/perf/perf"
+VMLINUX_FLAG = " --vmlinux=/usr/lib/debug/boot/vmlinux"
+PERF_CMD = PERF_BINARY +" report -i %s -n --symfs=%s" + VMLINUX_FLAG
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option("--in", dest="in_dir")
+  parser.add_option("--out", dest="out_dir")
+  parser.add_option("--cache", dest="cache")
+  (opts, _) = parser.parse_args()
+  if not _ValidateOpts(opts):
+    return 1
+  else:
+    for filename in os.listdir(opts.in_dir):
+      try:
+        _DownloadSymbols(filename, opts.cache)
+        _PerfReport(filename, opts.in_dir, opts.out_dir, opts.cache)
+      except:
+        print "Exception caught. Continuing..."
+  return 0
+
+
+def _ValidateOpts(opts):
+  """Ensures all directories exist, before attempting to populate."""
+  if not os.path.exists(opts.in_dir):
+    print "Input directory doesn't exist."
+    return False
+  if not os.path.exists(opts.out_dir):
+    print "Output directory doesn't exist. Creating it..."
+    os.makedirs(opts.out_dir)
+  if not os.path.exists(opts.cache):
+    print "Cache directory doesn't exist."
+    return False
+  return True
+
+
+def _ParseFilename(filename, canonical=False):
+  """Returns a tuple (key, time, board, lsb_version).
+     If canonical is True, instead returns (database_key, board, canonical_vers)
+     canonical_vers includes the revision string.
+  """
+  key, time, board, vers = filename.split("~")
+  if canonical:
+    vers = misc.GetChromeOSVersionFromLSBVersion(vers)
+  return (key, time, board, vers)
+
+
+def _FormReleaseDir(board, version):
+  return "%s-release~%s" % (board, version)
+
+
+def _DownloadSymbols(filename, cache):
+  """ Incrementally downloads appropriate symbols.
+      We store the downloads in cache, with each set of symbols in a TLD
+      named like cache/$board-release~$canonical_vers/usr/lib/debug
+  """
+  _, _, board, vers = _ParseFilename(filename, canonical=True)
+  tmp_suffix = ".tmp"
+
+  tarball_subdir = _FormReleaseDir(board, vers)
+  tarball_dir = os.path.join(cache, tarball_subdir)
+  tarball_path = os.path.join(tarball_dir, "debug.tgz")
+
+  symbol_subdir = os.path.join("usr", "lib")
+  symbol_dir = os.path.join(tarball_dir, symbol_subdir)
+
+  if os.path.isdir(symbol_dir):
+    print "Symbol directory %s exists, skipping download." % symbol_dir
+    return
+  else:
+    # First download using gsutil.
+    if not os.path.isfile(tarball_path):
+      download_cmd = GSUTIL_CMD % (board, vers, tarball_path + tmp_suffix)
+      print "Downloading symbols for %s" % filename
+      print download_cmd
+      ret = call(download_cmd.split())
+      if ret != 0:
+        print "gsutil returned non-zero error code: %s." % ret
+        # Clean up the empty directory structures.
+        os.remove(tarball_path + tmp_suffix)
+        raise IOError
+
+      shutil.move(tarball_path + tmp_suffix, tarball_path)
+
+    # Next, untar the tarball.
+    os.makedirs(symbol_dir + tmp_suffix)
+    extract_cmd = TAR_CMD % (tarball_path, symbol_dir + tmp_suffix)
+    print "Extracting symbols for %s" % filename
+    print extract_cmd
+    ret = call(extract_cmd.split())
+    if ret != 0:
+      print "tar returned non-zero code: %s." % ret
+      raise IOError
+    shutil.move(symbol_dir + tmp_suffix, symbol_dir)
+    os.remove(tarball_path)
+
+
+def _PerfReport(filename, in_dir, out_dir, cache):
+  """ Call perf report on the file, storing output to the output dir.
+      The output is currently stored as $out_dir/$filename
+  """
+  _, _, board, vers = _ParseFilename(filename, canonical=True)
+  symbol_cache_tld = _FormReleaseDir(board, vers)
+  input_file = os.path.join(in_dir, filename)
+  symfs = os.path.join(cache, symbol_cache_tld)
+  report_cmd = PERF_CMD % (input_file, symfs)
+  print "Reporting."
+  print report_cmd
+  report_proc = Popen(report_cmd.split(), stdout=PIPE)
+  outfile = open(os.path.join(out_dir, filename), "w")
+  outfile.write(report_proc.stdout.read())
+  outfile.close()
+
+
+if __name__ == "__main__":
+  exit(main())
diff --git a/cwp/performance/experiment_gen.py b/cwp/performance/experiment_gen.py
new file mode 100644
index 00000000..7752c11e
--- /dev/null
+++ b/cwp/performance/experiment_gen.py
@@ -0,0 +1,116 @@
+#!/usr/bin/python
+# Copyright 2012 Google Inc. All Rights Reserved.
+"""This script generates a crosperf overhead-testing experiment file for MoreJS.
+
+Use: experiment_gen.py --crosperf=/home/mrdmnd/depot2/crosperf --chromeos_root=
+/home/mrdmnd/chromiumos --remote-host=chromeos-zgb3.mtv --board=x86-zgb --event=
+cycles -F 10 -F 20 -c 10582 -c 10785211 --perf_options="-g"
+"""
+
+import optparse
+import subprocess
+import sys
+import time
+
+HEADER = """
+board: %s
+remote: %s
+benchmark: baseline {
+ iterations: %s
+ autotest_name: desktopui_PyAutoPerfTests
+ autotest_args: --args='--iterations=%s perf.PageCyclerTest.testMoreJSFile'
+}"""
+
+EXPERIMENT = """
+benchmark: %s {
+ iterations: %s
+ autotest_name: desktopui_PyAutoPerfTests
+ autotest_args: --args='--iterations=%s perf.PageCyclerTest.testMoreJSFile' --profiler=custom_perf --profiler_args='perf_options="record -a %s %s -e %s"' \n}"""  # pylint: disable-msg=C6310
+
+DEFAULT_IMAGE = """
+default {
+ chromeos_image: %s/src/build/images/%s/latest/chromiumos_test_image.bin
+}"""
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('--crosperf', dest='crosperf_root', action='store',
+                    default='/home/mrdmnd/depot2/crosperf',
+                    help='Crosperf root directory.')
+  parser.add_option('--chromeos_root', dest='chromeos_root', action='store',
+                    default='/home/mrdmnd/chromiumos',
+                    help='ChromiumOS root directory.')
+  parser.add_option('--remote', dest='remote', action='store',
+                    help='Host to run test on. Required.')
+  parser.add_option('--board', dest='board', action='store',
+                    help='Board architecture to run on. Required.')
+  parser.add_option('--event', dest='event', action='store',
+                    help='Event to profile. Required.')
+  parser.add_option('-F', dest='sampling_frequencies', action='append',
+                    help='A target frequency to sample at.')
+  parser.add_option('-c', dest='sampling_periods', action='append',
+                    help='A target period to sample at. Event specific.')
+  parser.add_option('--benchmark-iterations', dest='benchmark_iterations',
+                    action='store', default=4, help='Number of benchmark iters')
+  parser.add_option('--test-iterations', dest='test_iterations',
+                    action='store', default=10, help='Number of test iters')
+  parser.add_option('-p', dest='print_only', action='store_true',
+                    help='If enabled, will print experiment file and exit.')
+  parser.add_option('--perf_options', dest='perf_options', action='store',
+                    help='Arbitrary flags to perf. Surround with dblquotes.')
+  options = parser.parse_args()[0]
+  if options.remote is None:
+    print '%s requires a remote hostname.' % sys.argv[0]
+    return 1
+  elif options.board is None:
+    print '%s requires a target board.' % sys.argv[0]
+    return 1
+  elif options.event is None:
+    print '%s requires an event to profile.' % sys.argv[0]
+    return 1
+  else:
+    crosperf_root = options.crosperf_root
+    chromeos_root = options.chromeos_root
+    remote = options.remote
+    board = options.board
+    event = options.event
+    bench_iters = options.benchmark_iterations
+    test_iters = options.test_iterations
+    perf_opts = options.perf_options
+    # Set up baseline test.
+    experiment_file = HEADER % (board, remote, bench_iters, test_iters)
+    # Set up experimental tests.
+    if options.sampling_frequencies:
+      for freq in options.sampling_frequencies:
+        test_string = str(freq) + 'Freq'
+        experiment_file += EXPERIMENT % (test_string, bench_iters, test_iters,
+                                         '-F %s' % freq,
+                                         '' if perf_opts is None else perf_opts,
+                                         event)
+    if options.sampling_periods:
+      for period in options.sampling_periods:
+        test_string = str(period) + 'Period'
+        experiment_file += EXPERIMENT % (test_string, bench_iters, test_iters,
+                                         '-c %s' % period,
+                                         '' if perf_opts is None else perf_opts,
+                                         event)
+    # Point to the target image.
+    experiment_file += DEFAULT_IMAGE % (chromeos_root, board)
+    if options.print_only:
+      print experiment_file
+    else:
+      current_time = int(round(time.time() * 1000))
+      file_name = 'perf_overhead_%s' % str(current_time)
+      with open(file_name, "w") as f:
+        f.write(experiment_file)
+      try:
+        process = subprocess.Popen(['%s/crosperf' % crosperf_root, file_name])
+        process.communicate()
+      except OSError:
+        print 'Could not find crosperf, make sure --crosperf flag is set right.'
+        return 1
+    return 0
+
+if __name__ == '__main__':
+  exit(main())
author	Luis Lozano <llozano@chromium.org>	2013-03-15 14:44:13 -0700
committer	ChromeBot <chrome-bot@google.com>	2013-03-15 15:51:37 -0700
commit	f81680c018729fd4499e1e200d04b48c4b90127c (patch)
tree	940608da8374604b82edfdb2d7df55d065f05d4c /cwp
parent	2296ee0b914aba5bba07becab4ff68884ce9b8a5 (diff)
download	toolchain-utils-f81680c018729fd4499e1e200d04b48c4b90127c.tar.gz