diff options
Diffstat (limited to 'deprecated/cwp')
| -rw-r--r-- | deprecated/cwp/bartlett/app.yaml | 22 | ||||
| -rwxr-xr-x | deprecated/cwp/bartlett/server.py | 153 | ||||
| -rw-r--r-- | deprecated/cwp/bartlett/static/favicon.ico | bin | 0 -> 198 bytes | |||
| -rw-r--r-- | deprecated/cwp/bartlett/test/server_tester.py | 101 | ||||
| -rwxr-xr-x | deprecated/cwp/bartlett/update_appengine_server | 1 | ||||
| -rw-r--r-- | deprecated/cwp/demo_pipeline.sh | 55 | ||||
| -rw-r--r-- | deprecated/cwp/interpreter/app_engine_pull.py | 253 | ||||
| -rw-r--r-- | deprecated/cwp/interpreter/symbolizer.py | 129 | ||||
| -rw-r--r-- | deprecated/cwp/performance/experiment_gen.py | 138 |
9 files changed, 852 insertions, 0 deletions
diff --git a/deprecated/cwp/bartlett/app.yaml b/deprecated/cwp/bartlett/app.yaml new file mode 100644 index 00000000..60010f70 --- /dev/null +++ b/deprecated/cwp/bartlett/app.yaml @@ -0,0 +1,22 @@ +application: chromeoswideprofiling +version: 1 +runtime: python +api_version: 1 + +handlers: +- url: /favicon.ico + static_files: static/favicon.ico + upload: static/favicon.ico +- url: /remote_api + script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py + login: admin +- url: / + script: server.py +- url: /upload + script: server.py +- url: /serve + script: server.py +- url: /serve/.* + script: server.py +- url: /del/.* + script: server.py diff --git a/deprecated/cwp/bartlett/server.py b/deprecated/cwp/bartlett/server.py new file mode 100755 index 00000000..8fb7d61e --- /dev/null +++ b/deprecated/cwp/bartlett/server.py @@ -0,0 +1,153 @@ +#!/usr/bin/python2 +# Copyright 2012 Google Inc. All Rights Reserved. +# Author: mrdmnd@ (Matt Redmond) +# Based off of code in //depot/google3/experimental/mobile_gwp +"""Code to transport profile data between a user's machine and the CWP servers. + Pages: + "/": the main page for the app, left blank so that users cannot access + the file upload but left in the code for debugging purposes + "/upload": Updates the datastore with a new file. the upload depends on + the format which is templated on the main page ("/") + input includes: + profile_data: the zipped file containing profile data + board: the architecture we ran on + chromeos_version: the chromeos_version + "/serve": Lists all of the files in the datastore. Each line is a new entry + in the datastore. The format is key~date, where key is the entry's + key in the datastore and date is the file upload time and date. + (Authentication Required) + "/serve/([^/]+)?": For downloading a file of profile data, ([^/]+)? means + any character sequence so to download the file go to + '/serve/$key' where $key is the datastore key of the file + you want to download. + (Authentication Required) + "/del/([^/]+)?": For deleting an entry in the datastore. To use go to + '/del/$key' where $key is the datastore key of the entry + you want to be deleted form the datastore. + (Authentication Required) + TODO: Add more extensive logging""" + +import cgi +import logging +import md5 +import urllib + +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp.util import run_wsgi_app + +logging.getLogger().setLevel(logging.DEBUG) + + +class FileEntry(db.Model): + profile_data = db.BlobProperty() # The profile data + date = db.DateTimeProperty(auto_now_add=True) # Date it was uploaded + data_md5 = db.ByteStringProperty() # md5 of the profile data + board = db.StringProperty() # board arch + chromeos_version = db.StringProperty() # ChromeOS version + + +class MainPage(webapp.RequestHandler): + """Main page only used as the form template, not actually displayed.""" + + def get(self, response=''): # pylint: disable-msg=C6409 + if response: + self.response.out.write('<html><body>') + self.response.out.write("""<br> + <form action="/upload" enctype="multipart/form-data" method="post"> + <div><label>Profile Data:</label></div> + <div><input type="file" name="profile_data"/></div> + <div><label>Board</label></div> + <div><input type="text" name="board"/></div> + <div><label>ChromeOS Version</label></div> + <div><input type="text" name="chromeos_version"></div> + <div><input type="submit" value="send" name="submit"></div> + </form> + </body> + </html>""") + + +class Upload(webapp.RequestHandler): + """Handler for uploading data to the datastore, accessible by anyone.""" + + def post(self): # pylint: disable-msg=C6409 + """Takes input based on the main page's form.""" + getfile = FileEntry() + f1 = self.request.get('profile_data') + getfile.profile_data = db.Blob(f1) + getfile.data_md5 = md5.new(f1).hexdigest() + getfile.board = self.request.get('board') + getfile.chromeos_version = self.request.get('chromeos_version') + getfile.put() + self.response.out.write(getfile.key()) + #self.redirect('/') + + +class ServeHandler(webapp.RequestHandler): + """Given the entry's key in the database, output the profile data file. Only + accessible from @google.com accounts.""" + + def get(self, resource): # pylint: disable-msg=C6409 + if Authenticate(self): + file_key = str(urllib.unquote(resource)) + request = db.get(file_key) + self.response.out.write(request.profile_data) + + +class ListAll(webapp.RequestHandler): + """Displays all files uploaded. Only accessible by @google.com accounts.""" + + def get(self): # pylint: disable-msg=C6409 + """Displays all information in FileEntry, ~ delimited.""" + if Authenticate(self): + query_str = 'SELECT * FROM FileEntry ORDER BY date ASC' + query = db.GqlQuery(query_str) + delimiter = '~' + + for item in query: + display_list = [item.key(), item.date, item.data_md5, item.board, + item.chromeos_version] + str_list = [cgi.escape(str(i)) for i in display_list] + self.response.out.write(delimiter.join(str_list) + '</br>') + + +class DelEntries(webapp.RequestHandler): + """Deletes entries. Only accessible from @google.com accounts.""" + + def get(self, resource): # pylint: disable-msg=C6409 + """A specific entry is deleted, when the key is given.""" + if Authenticate(self): + fkey = str(urllib.unquote(resource)) + request = db.get(fkey) + if request: + db.delete(fkey) + + +def Authenticate(webpage): + """Some urls are only accessible if logged in with a @google.com account.""" + user = users.get_current_user() + if user is None: + webpage.redirect(users.create_login_url(webpage.request.uri)) + elif user.email().endswith('@google.com'): + return True + else: + webpage.response.out.write('Not Authenticated') + return False + + +def main(): + application = webapp.WSGIApplication( + [ + ('/', MainPage), + ('/upload', Upload), + ('/serve/([^/]+)?', ServeHandler), + ('/serve', ListAll), + ('/del/([^/]+)?', DelEntries), + ], + debug=False) + run_wsgi_app(application) + + +if __name__ == '__main__': + main() diff --git a/deprecated/cwp/bartlett/static/favicon.ico b/deprecated/cwp/bartlett/static/favicon.ico Binary files differnew file mode 100644 index 00000000..19b58c2e --- /dev/null +++ b/deprecated/cwp/bartlett/static/favicon.ico diff --git a/deprecated/cwp/bartlett/test/server_tester.py b/deprecated/cwp/bartlett/test/server_tester.py new file mode 100644 index 00000000..585da43a --- /dev/null +++ b/deprecated/cwp/bartlett/test/server_tester.py @@ -0,0 +1,101 @@ +# Copyright 2012 Google Inc. All Rights Reserved. +# Author: mrdmnd@ (Matt Redmond) +"""A unit test for sending data to Bartlett. Requires poster module.""" + +import cookielib +import os +import signal +import subprocess +import tempfile +import time +import unittest +import urllib2 + +from poster.encode import multipart_encode +from poster.streaminghttp import register_openers + +SERVER_DIR = '../.' +SERVER_URL = 'http://localhost:8080/' +GET = '_ah/login?email=googler@google.com&action=Login&continue=%s' +AUTH_URL = SERVER_URL + GET + + +class ServerTest(unittest.TestCase): + """A unit test for the bartlett server. Tests upload, serve, and delete.""" + + def setUp(self): + """Instantiate the files and server needed to test upload functionality.""" + self._server_proc = LaunchLocalServer() + self._jar = cookielib.LWPCookieJar() + self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._jar)) + + # We need these files to not delete when closed, because we have to reopen + # them in read mode after we write and close them. + self.profile_data = tempfile.NamedTemporaryFile(delete=False) + + size = 16 * 1024 + self.profile_data.write(os.urandom(size)) + + def tearDown(self): + self.profile_data.close() + os.remove(self.profile_data.name) + os.kill(self._server_proc.pid, signal.SIGINT) + + def testIntegration(self): # pylint: disable-msg=C6409 + key = self._testUpload() + self._testListAll() + self._testServeKey(key) + self._testDelKey(key) + + def _testUpload(self): # pylint: disable-msg=C6409 + register_openers() + data = {'profile_data': self.profile_data, + 'board': 'x86-zgb', + 'chromeos_version': '2409.0.2012_06_08_1114'} + datagen, headers = multipart_encode(data) + request = urllib2.Request(SERVER_URL + 'upload', datagen, headers) + response = urllib2.urlopen(request).read() + self.assertTrue(response) + return response + + def _testListAll(self): # pylint: disable-msg=C6409 + request = urllib2.Request(AUTH_URL % (SERVER_URL + 'serve')) + response = self.opener.open(request).read() + self.assertTrue(response) + + def _testServeKey(self, key): # pylint: disable-msg=C6409 + request = urllib2.Request(AUTH_URL % (SERVER_URL + 'serve/' + key)) + response = self.opener.open(request).read() + self.assertTrue(response) + + def _testDelKey(self, key): # pylint: disable-msg=C6409 + # There is no response to a delete request. + # We will check the listAll page to ensure there is no data. + request = urllib2.Request(AUTH_URL % (SERVER_URL + 'del/' + key)) + response = self.opener.open(request).read() + request = urllib2.Request(AUTH_URL % (SERVER_URL + 'serve')) + response = self.opener.open(request).read() + self.assertFalse(response) + + +def LaunchLocalServer(): + """Launch and store an authentication cookie with a local server.""" + proc = subprocess.Popen( + ['dev_appserver.py', '--clear_datastore', SERVER_DIR], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for server to come up + while True: + time.sleep(1) + try: + request = urllib2.Request(SERVER_URL + 'serve') + response = urllib2.urlopen(request).read() + if response: + break + except urllib2.URLError: + continue + return proc + + +if __name__ == '__main__': + unittest.main() diff --git a/deprecated/cwp/bartlett/update_appengine_server b/deprecated/cwp/bartlett/update_appengine_server new file mode 100755 index 00000000..f3812057 --- /dev/null +++ b/deprecated/cwp/bartlett/update_appengine_server @@ -0,0 +1 @@ +appcfg.py --oauth2 update . diff --git a/deprecated/cwp/demo_pipeline.sh b/deprecated/cwp/demo_pipeline.sh new file mode 100644 index 00000000..d45c5c44 --- /dev/null +++ b/deprecated/cwp/demo_pipeline.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# These should be on the local filesystem. We'll be hitting it hard. +DATA_DIR=/usr/local/google/home/${USER}/ +SYMBOL_CACHE=${DATA_DIR}cache/ +REPORT_DIR=${DATA_DIR}reports/ +SAMPLE_DIR=${DATA_DIR}samples/ +RECORD_FILE=/tmp/profiles.rio +COLUMN_FILE=/tmp/profiles.cio +mkdir -p ${SYMBOL_CACHE} +mkdir -p ${REPORT_DIR} +mkdir -p ${SAMPLE_DIR} + +# Directory that has the scripts app_engine_pull.py and symbolizer.py +INTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/ +V14_DIR=$(dirname $(dirname ${INTERPRETER_DIR})) + +PYTHONPATH=$PYTHONPATH:$V14_DIR + +# Profile util binary +PROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util + +# mr-convert binary +MR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert + +CNS_LOC=/cns/ag-d/home/${USER}/profiledb/ + +# Protofile location +PROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto + +echo "0. Cleaning up old data." +rm /tmp/profiles.* +rm ${REPORT_DIR}* +rm ${SAMPLE_DIR}* + + +echo "Starting CWP Pipeline..." +echo "1. Pulling samples to local filesystem from server." +echo " For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm" +python ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR} +echo "2. Symbolizing samples to perf reports. Hold on..." + +python ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE} +echo "3. Loading reports into RecordIO format..." +# Will need to make append_dir more clever / incremental +${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR} +echo "Done." +echo "4. Converting records to columnio." +${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC} +echo "5. Uploading columnio to colossus." +fileutil cp -f /tmp/profiles.cio-* ${CNS_LOC} +echo "6. Let's try some dremel queries..." +echo " dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*" +echo " Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;" + diff --git a/deprecated/cwp/interpreter/app_engine_pull.py b/deprecated/cwp/interpreter/app_engine_pull.py new file mode 100644 index 00000000..d092e2a3 --- /dev/null +++ b/deprecated/cwp/interpreter/app_engine_pull.py @@ -0,0 +1,253 @@ +# Copyright 2012 Google Inc. All Rights Reserved. +# Author: mrdmnd@ (Matt Redmond) +"""A client to pull data from Bartlett. + +Inspired by //depot/google3/experimental/mobile_gwp/database/app_engine_pull.py + +The server houses perf.data.gz, board, chrome version for each upload. +This script first authenticates with a proper @google.com account, then +downloads a sample (if it's not already cached) and unzips perf.data + + Authenticate(): Gets login info and returns an auth token + DownloadSamples(): Download and unzip samples. + _GetServePage(): Pulls /serve page from the app engine server + _DownloadSampleFromServer(): Downloads a local compressed copy of a sample + _UncompressSample(): Decompresses a sample, deleting the compressed version. +""" +import cookielib +import getpass +import gzip +import optparse +import os +import urllib +import urllib2 + +SERVER_NAME = 'http://chromeoswideprofiling.appspot.com' +APP_NAME = 'chromeoswideprofiling' +DELIMITER = '~' + + +def Authenticate(server_name): + """Gets credentials from user and attempts to retrieve auth token. + TODO: Accept OAuth2 instead of password. + Args: + server_name: (string) URL that the app engine code is living on. + Returns: + authtoken: (string) The authorization token that can be used + to grab other pages. + """ + + if server_name.endswith('/'): + server_name = server_name.rstrip('/') + # Grab username and password from user through stdin. + username = raw_input('Email (must be @google.com account): ') + password = getpass.getpass('Password: ') + # Use a cookie to authenticate with GAE. + cookiejar = cookielib.LWPCookieJar() + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) + urllib2.install_opener(opener) + # Get an AuthToken from Google accounts service. + auth_uri = 'https://www.google.com/accounts/ClientLogin' + authreq_data = urllib.urlencode({'Email': username, + 'Passwd': password, + 'service': 'ah', + 'source': APP_NAME, + 'accountType': 'HOSTED_OR_GOOGLE'}) + auth_req = urllib2.Request(auth_uri, data=authreq_data) + try: + auth_resp = urllib2.urlopen(auth_req) + except urllib2.URLError: + print 'Error logging in to Google accounts service.' + return None + body = auth_resp.read() + # Auth response contains several fields. + # We care about the part after Auth= + auth_resp_dict = dict(x.split('=') for x in body.split('\n') if x) + authtoken = auth_resp_dict['Auth'] + return authtoken + + +def DownloadSamples(server_name, authtoken, output_dir, start, stop): + """Download every sample and write unzipped version + to output directory. + Args: + server_name: (string) URL that the app engine code is living on. + authtoken: (string) Authorization token. + output_dir (string) Filepath to write output to. + start: (int) Index to start downloading from, starting at top. + stop: (int) Index to stop downloading, non-inclusive. -1 for end. + Returns: + None + """ + + if server_name.endswith('/'): + server_name = server_name.rstrip('/') + + serve_page_string = _GetServePage(server_name, authtoken) + if serve_page_string is None: + print 'Error getting /serve page.' + return + + sample_list = serve_page_string.split('</br>') + print 'Will download:' + sample_list_subset = sample_list[start:stop] + for sample in sample_list_subset: + print sample + for sample in sample_list_subset: + assert sample, 'Sample should be valid.' + sample_info = [s.strip() for s in sample.split(DELIMITER)] + key = sample_info[0] + time = sample_info[1] + time = time.replace(' ', '_') # No space between date and time. + # sample_md5 = sample_info[2] + board = sample_info[3] + version = sample_info[4] + + # Put a compressed copy of the samples in output directory. + _DownloadSampleFromServer(server_name, authtoken, key, time, board, version, + output_dir) + _UncompressSample(key, time, board, version, output_dir) + + +def _BuildFilenameFromParams(key, time, board, version): + """Return the filename for our sample. + Args: + key: (string) Key indexing our sample in the datastore. + time: (string) Date that the sample was uploaded. + board: (string) Board that the sample was taken on. + version: (string) Version string from /etc/lsb-release + Returns: + filename (string) + """ + filename = DELIMITER.join([key, time, board, version]) + return filename + + +def _DownloadSampleFromServer(server_name, authtoken, key, time, board, version, + output_dir): + """Downloads sample_$(samplekey).gz to current dir. + Args: + server_name: (string) URL that the app engine code is living on. + authtoken: (string) Authorization token. + key: (string) Key indexing our sample in the datastore + time: (string) Date that the sample was uploaded. + board: (string) Board that the sample was taken on. + version: (string) Version string from /etc/lsb-release + output_dir: (string) Filepath to write to output to. + Returns: + None + """ + filename = _BuildFilenameFromParams(key, time, board, version) + compressed_filename = filename + '.gz' + + if os.path.exists(os.path.join(output_dir, filename)): + print 'Already downloaded %s, skipping.' % filename + return + + serv_uri = server_name + '/serve/' + key + serv_args = {'continue': serv_uri, 'auth': authtoken} + full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) + serv_req = urllib2.Request(full_serv_uri) + serv_resp = urllib2.urlopen(serv_req) + f = open(os.path.join(output_dir, compressed_filename), 'w+') + f.write(serv_resp.read()) + f.close() + + +def _UncompressSample(key, time, board, version, output_dir): + """Uncompresses a given sample.gz file and deletes the compressed version. + Args: + key: (string) Sample key to uncompress. + time: (string) Date that the sample was uploaded. + board: (string) Board that the sample was taken on. + version: (string) Version string from /etc/lsb-release + output_dir: (string) Filepath to find sample key in. + Returns: + None + """ + filename = _BuildFilenameFromParams(key, time, board, version) + compressed_filename = filename + '.gz' + + if os.path.exists(os.path.join(output_dir, filename)): + print 'Already decompressed %s, skipping.' % filename + return + + out_file = open(os.path.join(output_dir, filename), 'wb') + in_file = gzip.open(os.path.join(output_dir, compressed_filename), 'rb') + out_file.write(in_file.read()) + in_file.close() + out_file.close() + os.remove(os.path.join(output_dir, compressed_filename)) + + +def _DeleteSampleFromServer(server_name, authtoken, key): + """Opens the /delete page with the specified key + to delete the sample off the datastore. + Args: + server_name: (string) URL that the app engine code is living on. + authtoken: (string) Authorization token. + key: (string) Key to delete. + Returns: + None + """ + + serv_uri = server_name + '/del/' + key + serv_args = {'continue': serv_uri, 'auth': authtoken} + full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) + serv_req = urllib2.Request(full_serv_uri) + urllib2.urlopen(serv_req) + + +def _GetServePage(server_name, authtoken): + """Opens the /serve page and lists all keys. + Args: + server_name: (string) URL the app engine code is living on. + authtoken: (string) Authorization token. + Returns: + The text of the /serve page (including HTML tags) + """ + + serv_uri = server_name + '/serve' + serv_args = {'continue': serv_uri, 'auth': authtoken} + full_serv_uri = server_name + '/_ah/login?%s' % urllib.urlencode(serv_args) + serv_req = urllib2.Request(full_serv_uri) + serv_resp = urllib2.urlopen(serv_req) + return serv_resp.read() + + +def main(): + parser = optparse.OptionParser() + parser.add_option('--output_dir', + dest='output_dir', + action='store', + help='Path to output perf data files.') + parser.add_option('--start', + dest='start_ind', + action='store', + default=0, + help='Start index.') + parser.add_option('--stop', + dest='stop_ind', + action='store', + default=-1, + help='Stop index.') + options = parser.parse_args()[0] + if not options.output_dir: + print 'Must specify --output_dir.' + return 1 + if not os.path.exists(options.output_dir): + print 'Specified output_dir does not exist.' + return 1 + + authtoken = Authenticate(SERVER_NAME) + if not authtoken: + print 'Could not obtain authtoken, exiting.' + return 1 + DownloadSamples(SERVER_NAME, authtoken, options.output_dir, options.start_ind, + options.stop_ind) + print 'Downloaded samples.' + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/deprecated/cwp/interpreter/symbolizer.py b/deprecated/cwp/interpreter/symbolizer.py new file mode 100644 index 00000000..4ece480d --- /dev/null +++ b/deprecated/cwp/interpreter/symbolizer.py @@ -0,0 +1,129 @@ +# Copyright 2012 Google Inc. All Rights Reserved. +"""A script that symbolizes perf.data files.""" +import optparse +import os +import shutil +from subprocess import call +from subprocess import PIPE +from subprocess import Popen +from cros_utils import misc + +GSUTIL_CMD = 'gsutil cp gs://chromeos-image-archive/%s-release/%s/debug.tgz %s' +TAR_CMD = 'tar -zxvf %s -C %s' +PERF_BINARY = '/google/data/ro/projects/perf/perf' +VMLINUX_FLAG = ' --vmlinux=/usr/lib/debug/boot/vmlinux' +PERF_CMD = PERF_BINARY + ' report -i %s -n --symfs=%s' + VMLINUX_FLAG + + +def main(): + parser = optparse.OptionParser() + parser.add_option('--in', dest='in_dir') + parser.add_option('--out', dest='out_dir') + parser.add_option('--cache', dest='cache') + (opts, _) = parser.parse_args() + if not _ValidateOpts(opts): + return 1 + else: + for filename in os.listdir(opts.in_dir): + try: + _DownloadSymbols(filename, opts.cache) + _PerfReport(filename, opts.in_dir, opts.out_dir, opts.cache) + except: + print 'Exception caught. Continuing...' + return 0 + + +def _ValidateOpts(opts): + """Ensures all directories exist, before attempting to populate.""" + if not os.path.exists(opts.in_dir): + print "Input directory doesn't exist." + return False + if not os.path.exists(opts.out_dir): + print "Output directory doesn't exist. Creating it..." + os.makedirs(opts.out_dir) + if not os.path.exists(opts.cache): + print "Cache directory doesn't exist." + return False + return True + + +def _ParseFilename(filename, canonical=False): + """Returns a tuple (key, time, board, lsb_version). + If canonical is True, instead returns (database_key, board, canonical_vers) + canonical_vers includes the revision string. + """ + key, time, board, vers = filename.split('~') + if canonical: + vers = misc.GetChromeOSVersionFromLSBVersion(vers) + return (key, time, board, vers) + + +def _FormReleaseDir(board, version): + return '%s-release~%s' % (board, version) + + +def _DownloadSymbols(filename, cache): + """ Incrementally downloads appropriate symbols. + We store the downloads in cache, with each set of symbols in a TLD + named like cache/$board-release~$canonical_vers/usr/lib/debug + """ + _, _, board, vers = _ParseFilename(filename, canonical=True) + tmp_suffix = '.tmp' + + tarball_subdir = _FormReleaseDir(board, vers) + tarball_dir = os.path.join(cache, tarball_subdir) + tarball_path = os.path.join(tarball_dir, 'debug.tgz') + + symbol_subdir = os.path.join('usr', 'lib') + symbol_dir = os.path.join(tarball_dir, symbol_subdir) + + if os.path.isdir(symbol_dir): + print 'Symbol directory %s exists, skipping download.' % symbol_dir + return + else: + # First download using gsutil. + if not os.path.isfile(tarball_path): + download_cmd = GSUTIL_CMD % (board, vers, tarball_path + tmp_suffix) + print 'Downloading symbols for %s' % filename + print download_cmd + ret = call(download_cmd.split()) + if ret != 0: + print 'gsutil returned non-zero error code: %s.' % ret + # Clean up the empty directory structures. + os.remove(tarball_path + tmp_suffix) + raise IOError + + shutil.move(tarball_path + tmp_suffix, tarball_path) + + # Next, untar the tarball. + os.makedirs(symbol_dir + tmp_suffix) + extract_cmd = TAR_CMD % (tarball_path, symbol_dir + tmp_suffix) + print 'Extracting symbols for %s' % filename + print extract_cmd + ret = call(extract_cmd.split()) + if ret != 0: + print 'tar returned non-zero code: %s.' % ret + raise IOError + shutil.move(symbol_dir + tmp_suffix, symbol_dir) + os.remove(tarball_path) + + +def _PerfReport(filename, in_dir, out_dir, cache): + """ Call perf report on the file, storing output to the output dir. + The output is currently stored as $out_dir/$filename + """ + _, _, board, vers = _ParseFilename(filename, canonical=True) + symbol_cache_tld = _FormReleaseDir(board, vers) + input_file = os.path.join(in_dir, filename) + symfs = os.path.join(cache, symbol_cache_tld) + report_cmd = PERF_CMD % (input_file, symfs) + print 'Reporting.' + print report_cmd + report_proc = Popen(report_cmd.split(), stdout=PIPE) + outfile = open(os.path.join(out_dir, filename), 'w') + outfile.write(report_proc.stdout.read()) + outfile.close() + + +if __name__ == '__main__': + exit(main()) diff --git a/deprecated/cwp/performance/experiment_gen.py b/deprecated/cwp/performance/experiment_gen.py new file mode 100644 index 00000000..a12da2c5 --- /dev/null +++ b/deprecated/cwp/performance/experiment_gen.py @@ -0,0 +1,138 @@ +# Copyright 2012 Google Inc. All Rights Reserved. +"""This script generates a crosperf overhead-testing experiment file for MoreJS. + +Use: experiment_gen.py --crosperf=/home/mrdmnd/depot2/crosperf --chromeos_root= +/home/mrdmnd/chromiumos --remote-host=chromeos-zgb3.mtv --board=x86-zgb --event= +cycles -F 10 -F 20 -c 10582 -c 10785211 --perf_options="-g" +""" + +import optparse +import subprocess +import sys +import time + +HEADER = """ +board: %s +remote: %s +benchmark: baseline { + iterations: %s + autotest_name: desktopui_PyAutoPerfTests + autotest_args: --args='--iterations=%s perf.PageCyclerTest.testMoreJSFile' +}""" + +EXPERIMENT = """ +benchmark: %s { + iterations: %s + autotest_name: desktopui_PyAutoPerfTests + autotest_args: --args='--iterations=%s perf.PageCyclerTest.testMoreJSFile' --profiler=custom_perf --profiler_args='perf_options="record -a %s %s -e %s"' \n}""" # pylint: disable-msg=C6310 + +DEFAULT_IMAGE = """ +default { + chromeos_image: %s/src/build/images/%s/latest/chromiumos_test_image.bin +}""" + + +def main(): + parser = optparse.OptionParser() + parser.add_option('--crosperf', + dest='crosperf_root', + action='store', + default='/home/mrdmnd/depot2/crosperf', + help='Crosperf root directory.') + parser.add_option('--chromeos_root', + dest='chromeos_root', + action='store', + default='/home/mrdmnd/chromiumos', + help='ChromiumOS root directory.') + parser.add_option('--remote', + dest='remote', + action='store', + help='Host to run test on. Required.') + parser.add_option('--board', + dest='board', + action='store', + help='Board architecture to run on. Required.') + parser.add_option('--event', + dest='event', + action='store', + help='Event to profile. Required.') + parser.add_option('-F', + dest='sampling_frequencies', + action='append', + help='A target frequency to sample at.') + parser.add_option('-c', + dest='sampling_periods', + action='append', + help='A target period to sample at. Event specific.') + parser.add_option('--benchmark-iterations', + dest='benchmark_iterations', + action='store', + default=4, + help='Number of benchmark iters') + parser.add_option('--test-iterations', + dest='test_iterations', + action='store', + default=10, + help='Number of test iters') + parser.add_option('-p', + dest='print_only', + action='store_true', + help='If enabled, will print experiment file and exit.') + parser.add_option('--perf_options', + dest='perf_options', + action='store', + help='Arbitrary flags to perf. Surround with dblquotes.') + options = parser.parse_args()[0] + if options.remote is None: + print '%s requires a remote hostname.' % sys.argv[0] + return 1 + elif options.board is None: + print '%s requires a target board.' % sys.argv[0] + return 1 + elif options.event is None: + print '%s requires an event to profile.' % sys.argv[0] + return 1 + else: + crosperf_root = options.crosperf_root + chromeos_root = options.chromeos_root + remote = options.remote + board = options.board + event = options.event + bench_iters = options.benchmark_iterations + test_iters = options.test_iterations + perf_opts = options.perf_options + # Set up baseline test. + experiment_file = HEADER % (board, remote, bench_iters, test_iters) + # Set up experimental tests. + if options.sampling_frequencies: + for freq in options.sampling_frequencies: + test_string = str(freq) + 'Freq' + experiment_file += EXPERIMENT % (test_string, bench_iters, test_iters, + '-F %s' % freq, '' if perf_opts is None + else perf_opts, event) + if options.sampling_periods: + for period in options.sampling_periods: + test_string = str(period) + 'Period' + experiment_file += EXPERIMENT % ( + test_string, bench_iters, test_iters, '-c %s' % period, '' if + perf_opts is None else perf_opts, event) + # Point to the target image. + experiment_file += DEFAULT_IMAGE % (chromeos_root, board) + if options.print_only: + print experiment_file + else: + current_time = int(round(time.time() * 1000)) + file_name = 'perf_overhead_%s' % str(current_time) + with open(file_name, 'w') as f: + f.write(experiment_file) + try: + process = subprocess.Popen(['%s/crosperf' % crosperf_root, file_name]) + process.communicate() + except OSError: + print 'Could not find crosperf, make sure --crosperf flag is set right.' + return 1 + return 0 + + +if __name__ == '__main__': + exit(main()) |